From 0a4c04d14eb56487c85504e6be5b547d24b5044d Mon Sep 17 00:00:00 2001 From: Diego Sampaio Date: Thu, 20 Apr 2017 17:25:37 -0300 Subject: [PATCH 01/42] Add S3 storage type to avatars --- .../client/lib/FileUploadAmazonS3.js | 33 +-- .../client/lib/fileUploadHandler.js | 4 +- .../globalFileRestrictions.js | 1 + .../server/config/configFileUploadAmazonS3.js | 148 ++++++----- .../server/lib/FileUpload.js | 2 +- .../server/models/Uploads.coffee | 53 +++- .../client/avatar/prompt.coffee | 34 ++- .../rocketchat-ui/client/lib/fileUpload.js | 232 ++++++++++++++++++ packages/rocketchat-ui/package.js | 4 +- server/methods/setAvatarFromService.js | 7 + server/startup/avatar.js | 147 ++++++++--- 11 files changed, 531 insertions(+), 134 deletions(-) create mode 100644 packages/rocketchat-ui/client/lib/fileUpload.js diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js b/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js index 277f6471a9aa..df9fa58e3f56 100644 --- a/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js +++ b/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js @@ -1,36 +1,19 @@ /* globals FileUpload, FileUploadBase, Slingshot */ FileUpload.AmazonS3 = class FileUploadAmazonS3 extends FileUploadBase { - constructor(meta, file) { + constructor(directive, meta, file) { super(meta, file); - this.uploader = new Slingshot.Upload('rocketchat-uploads', { rid: meta.rid }); + this.uploader = new Slingshot.Upload(directive, meta); } - start() { + start(callback) { this.uploader.send(this.file, (error, downloadUrl) => { if (this.computation) { this.computation.stop(); } if (error) { - let uploading = Session.get('uploading'); - if (!Array.isArray(uploading)) { - uploading = []; - } - - const item = _.findWhere(uploading, { id: this.id }); - - if (_.isObject(item)) { - item.error = error.error; - item.percentage = 0; - } else { - uploading.push({ - error: error.error, - percentage: 0 - }); - } - - Session.set('uploading', uploading); + return callback.call(this, error); } else { const file = _.pick(this.meta, 'type', 'size', 'name', 'identify', 'description'); file._id = downloadUrl.substr(downloadUrl.lastIndexOf('/') + 1); @@ -38,13 +21,7 @@ FileUpload.AmazonS3 = class FileUploadAmazonS3 extends FileUploadBase { Meteor.call('sendFileMessage', this.meta.rid, 's3', file, () => { Meteor.setTimeout(() => { - const uploading = Session.get('uploading'); - if (uploading !== null) { - const item = _.findWhere(uploading, { - id: this.id - }); - return Session.set('uploading', _.without(uploading, item)); - } + callback.call(this, null, file); }, 2000); }); } diff --git a/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js b/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js index 71f5fbdf1519..f62e4bb5b93b 100644 --- a/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js +++ b/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js @@ -1,10 +1,10 @@ /* globals FileUpload, fileUploadHandler:true */ /* exported fileUploadHandler */ -fileUploadHandler = (meta, file) => { +fileUploadHandler = (directive, meta, file) => { const storageType = RocketChat.settings.get('FileUpload_Storage_Type'); if (FileUpload[storageType] !== undefined) { - return new FileUpload[storageType](meta, file); + return new FileUpload[storageType](directive, meta, file); } }; diff --git a/packages/rocketchat-file-upload/globalFileRestrictions.js b/packages/rocketchat-file-upload/globalFileRestrictions.js index 0568c018d8f1..3c52390c4341 100644 --- a/packages/rocketchat-file-upload/globalFileRestrictions.js +++ b/packages/rocketchat-file-upload/globalFileRestrictions.js @@ -25,5 +25,6 @@ const slingShotConfig = { allowedFileTypes: null }; +Slingshot.fileRestrictions('rocketchat-avatars', slingShotConfig); Slingshot.fileRestrictions('rocketchat-uploads', slingShotConfig); Slingshot.fileRestrictions('rocketchat-uploads-gs', slingShotConfig); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js index daac0ea19932..284edb19921a 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js @@ -1,4 +1,4 @@ -/* globals Slingshot, FileUpload, AWS, SystemLogger */ +/* globals Slingshot, FileUpload, AWS */ const crypto = Npm.require('crypto'); let S3accessKey; @@ -36,8 +36,81 @@ FileUpload.addHandler('s3', { } }); -const createS3Directive = _.debounce(() => { - const directiveName = 'rocketchat-uploads'; +function createDirective(directiveName, { key, bucket, accessKey, secretKey, region, acl, cdn, bucketUrl}) { + if (Slingshot._directives[directiveName]) { + delete Slingshot._directives[directiveName]; + } + const config = { + bucket, + key, + AWSAccessKeyId: accessKey, + AWSSecretAccessKey: secretKey + }; + + if (!_.isEmpty(acl)) { + config.acl = acl; + } + + if (!_.isEmpty(cdn)) { + config.cdn = cdn; + } + + if (!_.isEmpty(region)) { + config.region = region; + } + + if (!_.isEmpty(bucketUrl)) { + config.bucketUrl = bucketUrl; + } + + try { + Slingshot.createDirective(directiveName, Slingshot.S3Storage, config); + } catch (e) { + console.error('Error configuring S3 ->', e.message); + } +} + +const configureSlingshot = _.debounce(() => { + const directives = [ + { + name: 'rocketchat-uploads', + key(file, metaContext) { + const path = `${ RocketChat.hostname }/${ metaContext.rid }/${ this.userId }/`; + + const upload = { + rid: metaContext.rid, + s3: { + bucket: RocketChat.settings.get('FileUpload_S3_Bucket'), + region: RocketChat.settings.get('FileUpload_S3_Region'), + path + } + }; + const fileId = RocketChat.models.Uploads.insertFileInit(this.userId, 's3', file, upload); + + return path + fileId; + } + }, + { + name: 'rocketchat-avatars', + key(file/*, metaContext*/) { + const path = `${ RocketChat.hostname }/avatars/`; + + const user = RocketChat.models.Users.findOneById(this.userId); + + const upload = { + username: user && user.username, + s3: { + bucket: RocketChat.settings.get('FileUpload_S3_Bucket'), + region: RocketChat.settings.get('FileUpload_S3_Region'), + path + } + }; + RocketChat.models.Uploads.insertFileInitByUsername(user.username, this.userId, 's3', file, upload); + + return path + user.username; + } + } + ]; const type = RocketChat.settings.get('FileUpload_Storage_Type'); const bucket = RocketChat.settings.get('FileUpload_S3_Bucket'); @@ -54,76 +127,41 @@ const createS3Directive = _.debounce(() => { }); if (type === 'AmazonS3' && !_.isEmpty(bucket) && !_.isEmpty(accessKey) && !_.isEmpty(secretKey)) { - if (Slingshot._directives[directiveName]) { - delete Slingshot._directives[directiveName]; - } - const config = { - bucket, - AWSAccessKeyId: accessKey, - AWSSecretAccessKey: secretKey, - key(file, metaContext) { - const path = `${ RocketChat.hostname }/${ metaContext.rid }/${ this.userId }/`; - - const upload = { s3: { - bucket, - region, - path - }}; - const fileId = RocketChat.models.Uploads.insertFileInit(metaContext.rid, this.userId, 's3', file, upload); - - return path + fileId; + directives.forEach((conf) => { + createDirective(conf.name, { key: conf.key, bucket, accessKey, secretKey, region, acl, cdn, bucketUrl}); + }); + } else { + directives.forEach((conf) => { + if (Slingshot._directives[conf.name]) { + delete Slingshot._directives[conf.name]; } - }; - - if (!_.isEmpty(acl)) { - config.acl = acl; - } - - if (!_.isEmpty(cdn)) { - config.cdn = cdn; - } - - if (!_.isEmpty(region)) { - config.region = region; - } - - if (!_.isEmpty(bucketUrl)) { - config.bucketUrl = bucketUrl; - } - - try { - Slingshot.createDirective(directiveName, Slingshot.S3Storage, config); - } catch (e) { - SystemLogger.error('Error configuring S3 ->', e.message); - } - } else if (Slingshot._directives[directiveName]) { - delete Slingshot._directives[directiveName]; + }); } }, 500); -RocketChat.settings.get('FileUpload_Storage_Type', createS3Directive); +RocketChat.settings.get('FileUpload_Storage_Type', configureSlingshot); -RocketChat.settings.get('FileUpload_S3_Bucket', createS3Directive); +RocketChat.settings.get('FileUpload_S3_Bucket', configureSlingshot); -RocketChat.settings.get('FileUpload_S3_Acl', createS3Directive); +RocketChat.settings.get('FileUpload_S3_Acl', configureSlingshot); RocketChat.settings.get('FileUpload_S3_AWSAccessKeyId', function(key, value) { S3accessKey = value; - createS3Directive(); + configureSlingshot(); }); RocketChat.settings.get('FileUpload_S3_AWSSecretAccessKey', function(key, value) { S3secretKey = value; - createS3Directive(); + configureSlingshot(); }); RocketChat.settings.get('FileUpload_S3_URLExpiryTimeSpan', function(key, value) { S3expiryTimeSpan = value; - createS3Directive(); + configureSlingshot(); }); -RocketChat.settings.get('FileUpload_S3_CDN', createS3Directive); +RocketChat.settings.get('FileUpload_S3_CDN', configureSlingshot); -RocketChat.settings.get('FileUpload_S3_Region', createS3Directive); +RocketChat.settings.get('FileUpload_S3_Region', configureSlingshot); -RocketChat.settings.get('FileUpload_S3_BucketURL', createS3Directive); +RocketChat.settings.get('FileUpload_S3_BucketURL', configureSlingshot); diff --git a/packages/rocketchat-file-upload/server/lib/FileUpload.js b/packages/rocketchat-file-upload/server/lib/FileUpload.js index cb0aadbc06c9..05ec504af3cc 100644 --- a/packages/rocketchat-file-upload/server/lib/FileUpload.js +++ b/packages/rocketchat-file-upload/server/lib/FileUpload.js @@ -16,7 +16,7 @@ FileUpload.delete = function(fileId) { this.handlers[file.store].delete(file); - return RocketChat.models.Uploads.remove(file._id); + return RocketChat.models.Uploads.deleteFile(file._id); }; FileUpload.get = function(file, req, res, next) { diff --git a/packages/rocketchat-lib/server/models/Uploads.coffee b/packages/rocketchat-lib/server/models/Uploads.coffee index 53dd79256891..2604403bbe6c 100644 --- a/packages/rocketchat-lib/server/models/Uploads.coffee +++ b/packages/rocketchat-lib/server/models/Uploads.coffee @@ -29,9 +29,8 @@ RocketChat.models.Uploads = new class extends RocketChat.models._Base return @find fileQuery, fileOptions - insertFileInit: (roomId, userId, store, file, extra) -> + insertFileInit: (userId, store, file, extra) -> fileData = - rid: roomId userId: userId store: store complete: false @@ -49,6 +48,23 @@ RocketChat.models.Uploads = new class extends RocketChat.models._Base return file + insertFileInitByUsername: (username, userId, store, file, extra) -> + fileData = + _id: username + userId: userId + store: store + complete: false + uploading: true + progress: 0 + extension: s.strRightBack(file.name, '.') + uploadedAt: new Date() + + _.extend(fileData, file, extra); + + file = @insertOrUpsert fileData + + return file + updateFileComplete: (fileId, userId, file) -> if not fileId return @@ -65,9 +81,40 @@ RocketChat.models.Uploads = new class extends RocketChat.models._Base update.$set = _.extend file, update.$set - if @model.direct?.insert? + if @model.direct?.update? + result = @model.direct.update filter, update + else + result = @update filter, update + + return result + + updateFileCompleteByUsername: (username, userId, url) -> + if not username + return + + filter = + username: username + userId: userId + + update = + $set: + complete: true + uploading: false + progress: 1 + url: url + + if @model.direct?.update? result = @model.direct.update filter, update else result = @update filter, update return result + + findOneByUsername: (username) -> + return @findOne username: username + + deleteFile: (fileId) -> + if @model.direct?.remove? + return @model.direct.remove { _id: fileId } + else + return @remove { _id: fileId } diff --git a/packages/rocketchat-ui-account/client/avatar/prompt.coffee b/packages/rocketchat-ui-account/client/avatar/prompt.coffee index 50f3d8838fdf..5b4849a1dcd2 100644 --- a/packages/rocketchat-ui-account/client/avatar/prompt.coffee +++ b/packages/rocketchat-ui-account/client/avatar/prompt.coffee @@ -1,4 +1,6 @@ import toastr from 'toastr' +import mime from 'mime-type/with-db' + Template.avatarPrompt.onCreated -> self = this self.suggestions = new ReactiveVar @@ -38,7 +40,7 @@ Template.avatarPrompt.helpers return '@'+Meteor.user()?.username Template.avatarPrompt.events - 'click .select-service': -> + 'click .select-service': (event, instance) -> if @service is 'initials' Meteor.call 'resetAvatar', (err) -> if err?.details?.timeToReset? @@ -60,13 +62,29 @@ Template.avatarPrompt.events else toastr.error t('Please_enter_value_for_url') else - tmpService = @service - Meteor.call 'setAvatarFromService', @blob, @contentType, @service, (err) -> - if err?.details?.timeToReset? - toastr.error t('error-too-many-requests', { seconds: parseInt(err.details.timeToReset / 1000) }) - else - toastr.success t('Avatar_changed_successfully') - RocketChat.callbacks.run('userAvatarSet', tmpService) + files = instance.find('input[type=file]').files + if not files or files.length is 0 + files = e.dataTransfer?.files or [] + + for file in files + Object.defineProperty(file, 'type', { value: mime.lookup(file.name) }) + + record = + name: files[0].name + size: files[0].size + type: files[0].type + # description: document.getElementById('file-description').value + + upload = fileUploadHandler 'rocketchat-avatars', record, files[0] + + # upload.onProgress = (progress) -> + # console.log 'progress ->', progress + + upload.start (error, result) -> + if result + Meteor.call 'saveAvatarFile', result, () => + toastr.success t('Avatar_changed_successfully') + RocketChat.callbacks.run('userAvatarSet', @service) 'click .login-with-service': (event, template) -> loginWithService = "loginWith#{_.capitalize(this)}" diff --git a/packages/rocketchat-ui/client/lib/fileUpload.js b/packages/rocketchat-ui/client/lib/fileUpload.js new file mode 100644 index 000000000000..ad91b1fc40e8 --- /dev/null +++ b/packages/rocketchat-ui/client/lib/fileUpload.js @@ -0,0 +1,232 @@ +/* globals fileUploadHandler, Handlebars, fileUpload */ +/* exported fileUpload */ + +function readAsDataURL(file, callback) { + const reader = new FileReader(); + reader.onload = ev => callback(ev.target.result, file); + + return reader.readAsDataURL(file); +} + +function getUploadPreview(file, callback) { + // If greater then 10MB don't try and show a preview + if (file.file.size > (10 * 1000000)) { + return callback(file, null); + } else if ((file.file.type.indexOf('audio') > -1) || (file.file.type.indexOf('video') > -1) || (file.file.type.indexOf('image') > -1)) { + file.type = file.file.type.split('/')[0]; + + return readAsDataURL(file.file, content => callback(file, content)); + } else { + return callback(file, null); + } +} + +function formatBytes(bytes, decimals) { + if (bytes === 0) { + return '0 Bytes'; + } + + const k = 1000; + const dm = (decimals + 1) || 3; + + const sizes = [ + 'Bytes', + 'KB', + 'MB', + 'GB', + 'TB', + 'PB' + ]; + + const i = Math.floor(Math.log(bytes) / Math.log(k)); + + return `${ parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) } ${ sizes[i] }`; +} + +fileUpload = function(filesToUpload) { + const roomId = Session.get('openedRoom'); + const files = [].concat(filesToUpload); + + function consume() { + const file = files.pop(); + if ((file == null)) { + swal.close(); + return; + } + + if (!RocketChat.fileUploadIsValidContentType(file.file.type)) { + swal({ + title: t('FileUpload_MediaType_NotAccepted'), + text: file.file.type || `*.${ s.strRightBack(file.file.name, '.') }`, + type: 'error', + timer: 3000 + }); + return; + } + + if (file.file.size === 0) { + swal({ + title: t('FileUpload_File_Empty'), + type: 'error', + timer: 1000 + }); + return; + } + + return getUploadPreview(file, function(file, preview) { + let text = ''; + + if (file.type === 'audio') { + text = `\ +
+ +
+
+ + +
`; + } else if (file.type === 'video') { + text = `\ +
+ +
+
+ + +
`; + } else if (file.type === 'image') { + text = `\ +
+
+
+
+ + +
`; + } else { + const fileSize = formatBytes(file.file.size); + + text = `\ +
+
${ Handlebars._escape(file.name) } - ${ fileSize }
+
+
+ + +
`; + } + + return swal({ + title: t('Upload_file_question'), + text, + showCancelButton: true, + closeOnConfirm: false, + closeOnCancel: false, + confirmButtonText: t('Send'), + cancelButtonText: t('Cancel'), + html: true + }, function(isConfirm) { + consume(); + if (isConfirm !== true) { + return; + } + + const record = { + name: document.getElementById('file-name').value || file.name || file.file.name, + size: file.file.size, + type: file.file.type, + rid: roomId, + description: document.getElementById('file-description').value + }; + + const upload = fileUploadHandler('rocketchat-uploads', record, file.file); + + let uploading = Session.get('uploading') || []; + uploading.push({ + id: upload.id, + name: upload.getFileName(), + percentage: 0 + }); + + Session.set('uploading', uploading); + + upload.onProgress = function(progress) { + uploading = Session.get('uploading'); + + const item = _.findWhere(uploading, {id: upload.id}); + if (item != null) { + item.percentage = Math.round(progress * 100) || 0; + return Session.set('uploading', uploading); + } + }; + + upload.start(function(error, file) { + if (error) { + let uploading = Session.get('uploading'); + if (!Array.isArray(uploading)) { + uploading = []; + } + + const item = _.findWhere(uploading, { id: this.id }); + + if (_.isObject(item)) { + item.error = error.error; + item.percentage = 0; + } else { + uploading.push({ + error: error.error, + percentage: 0 + }); + } + + Session.set('uploading', uploading); + return; + } else if (file) { + const uploading = Session.get('uploading'); + if (uploading !== null) { + const item = _.findWhere(uploading, { + id: this.id + }); + return Session.set('uploading', _.without(uploading, item)); + } + } + }); + + Tracker.autorun(function(c) { + const cancel = Session.get(`uploading-cancel-${ upload.id }`); + if (cancel) { + let item; + upload.stop(); + c.stop(); + + uploading = Session.get('uploading'); + if (uploading != null) { + item = _.findWhere(uploading, {id: upload.id}); + if (item != null) { + item.percentage = 0; + } + Session.set('uploading', uploading); + } + + return Meteor.setTimeout(function() { + uploading = Session.get('uploading'); + if (uploading != null) { + item = _.findWhere(uploading, {id: upload.id}); + return Session.set('uploading', _.without(uploading, item)); + } + } + , 1000); + } + }); + }); + }); + } + + consume(); +}; diff --git a/packages/rocketchat-ui/package.js b/packages/rocketchat-ui/package.js index 0a754855308e..acd455f1fe4d 100644 --- a/packages/rocketchat-ui/package.js +++ b/packages/rocketchat-ui/package.js @@ -38,7 +38,7 @@ Package.onUse(function(api) { api.addFiles('client/lib/chatMessages.coffee', 'client'); api.addFiles('client/lib/collections.js', 'client'); api.addFiles('client/lib/customEventPolyfill.js', 'client'); - api.addFiles('client/lib/fileUpload.coffee', 'client'); + api.addFiles('client/lib/fileUpload.js', 'client'); api.addFiles('client/lib/fireEvent.js', 'client'); api.addFiles('client/lib/iframeCommands.js', 'client'); api.addFiles('client/lib/menu.js', 'client'); @@ -108,4 +108,6 @@ Package.onUse(function(api) { api.addFiles('client/views/app/videoCall/videoButtons.js', 'client'); api.addFiles('client/views/app/videoCall/videoCall.js', 'client'); api.addFiles('client/views/app/photoswipe.js', 'client'); + + api.export('fileUpload'); }); diff --git a/server/methods/setAvatarFromService.js b/server/methods/setAvatarFromService.js index 6dc13b719e2e..bdc0768552c3 100644 --- a/server/methods/setAvatarFromService.js +++ b/server/methods/setAvatarFromService.js @@ -19,6 +19,13 @@ Meteor.methods({ const user = Meteor.user(); return RocketChat.setUserAvatar(user, dataURI, contentType, service); + }, + + saveAvatarFile(file) { + const user = RocketChat.models.Users.findOneById(Meteor.userId()); + // console.log('user ->', user); + // console.log('file ->', file); + RocketChat.models.Uploads.updateFileCompleteByUsername(user.username, Meteor.userId(), file.url); } }); diff --git a/server/startup/avatar.js b/server/startup/avatar.js index 6aca22c66592..634effe061b2 100644 --- a/server/startup/avatar.js +++ b/server/startup/avatar.js @@ -35,13 +35,11 @@ Meteor.startup(function() { transformWrite }); - return WebApp.connectHandlers.use('/avatar/', Meteor.bindEnvironment(function(req, res/*, next*/) { + WebApp.connectHandlers.use('/avatar/', Meteor.bindEnvironment(function(req, res/*, next*/) { const params = { username: decodeURIComponent(req.url.replace(/^\//, '').replace(/\?.*$/, '')) }; - let username = params.username.replace(/\.jpg$/, '').replace(/^@/, ''); - if (_.isEmpty(params.username)) { res.writeHead(403); res.write('Forbidden'); @@ -49,24 +47,23 @@ Meteor.startup(function() { return; } - let file; + return getNewAvatar(params, req, res); + })); +}); - if (params.username[0] !== '@') { - if (Meteor.settings && Meteor.settings.public && Meteor.settings.public.sandstorm) { - const user = RocketChat.models.Users.findOneByUsername(username); - if (user && user.services && user.services.sandstorm && user.services.sandstorm.picture) { - res.setHeader('Location', user.services.sandstorm.picture); - res.writeHead(302); - res.end(); - return; - } - } - file = RocketChatFileAvatarInstance.getFileWithReadStream(encodeURIComponent(`${ username }.jpg`)); - } +function getNewAvatar(params, req, res) { + const match = /^\/([^?]*)/.exec(req.url); + + if (match[1]) { + let username = decodeURIComponent(match[1]); + const file = RocketChat.models.Uploads.findOneByUsername(username); + + if (file) { + res.setHeader('Content-Security-Policy', 'default-src \'none\''); - res.setHeader('Content-Disposition', 'inline'); - if (!file) { + return FileUpload.get(file, req, res); + } else { res.setHeader('Content-Type', 'image/svg+xml'); res.setHeader('Cache-Control', 'public, max-age=0'); res.setHeader('Expires', '-1'); @@ -120,35 +117,113 @@ Meteor.startup(function() { return; } + } + + res.writeHead(404); + res.end(); + return; +} + +function getOldAvatar(params, req, res) { + let username = params.username.replace(/\.jpg$/, '').replace(/^@/, ''); + let file; + if (params.username[0] !== '@') { + if (Meteor.settings && Meteor.settings.public && Meteor.settings.public.sandstorm) { + const user = RocketChat.models.Users.findOneByUsername(username); + if (user && user.services && user.services.sandstorm && user.services.sandstorm.picture) { + res.setHeader('Location', user.services.sandstorm.picture); + res.writeHead(302); + res.end(); + return; + } + } + + file = RocketChatFileAvatarInstance.getFileWithReadStream(encodeURIComponent(`${ username }.jpg`)); + } + + res.setHeader('Content-Disposition', 'inline'); + if (!file) { + res.setHeader('Content-Type', 'image/svg+xml'); + res.setHeader('Cache-Control', 'public, max-age=0'); + res.setHeader('Expires', '-1'); + res.setHeader('Last-Modified', 'Thu, 01 Jan 2015 00:00:00 GMT'); const reqModifiedHeader = req.headers['if-modified-since']; + if (reqModifiedHeader) { - if (reqModifiedHeader === (file.uploadDate && file.uploadDate.toUTCString())) { - res.setHeader('Last-Modified', reqModifiedHeader); + if (reqModifiedHeader === 'Thu, 01 Jan 2015 00:00:00 GMT') { res.writeHead(304); res.end(); return; } } - res.setHeader('Cache-Control', 'public, max-age=0'); - res.setHeader('Expires', '-1'); - res.setHeader('Last-Modified', (file.uploadDate && file.uploadDate.toUTCString()) || new Date().toUTCString()); - res.setHeader('Content-Length', file.length); + const colors = ['#F44336', '#E91E63', '#9C27B0', '#673AB7', '#3F51B5', '#2196F3', '#03A9F4', '#00BCD4', '#009688', '#4CAF50', '#8BC34A', '#CDDC39', '#FFC107', '#FF9800', '#FF5722', '#795548', '#9E9E9E', '#607D8B']; - if (file.contentType) { - res.setHeader('Content-Type', file.contentType); - } else { - file.readStream.once('data', function(chunk) { - const fileType = getFileType(chunk); - if (fileType) { - return res.setHeader('Content-Type', fileType.mime); - } else { - return res.setHeader('Content-Type', 'image/jpeg'); + if (RocketChat.settings.get('UI_Use_Name_Avatar')) { + const user = RocketChat.models.Users.findOneByUsername(username, { + fields: { + name: 1 } }); + + if (user && user.name) { + username = user.name; + } } - file.readStream.pipe(res); - })); -}); + let color = ''; + let initials = ''; + + if (username === '?') { + color = '#000'; + initials = username; + } else { + const position = username.length % colors.length; + + color = colors[position]; + username = username.replace(/[^A-Za-z0-9]/g, '.').replace(/\.+/g, '.').replace(/(^\.)|(\.$)/g, ''); + + const usernameParts = username.split('.'); + + initials = usernameParts.length > 1 ? _.first(usernameParts)[0] + _.last(usernameParts)[0] : username.replace(/[^A-Za-z0-9]/g, '').substr(0, 2); + initials = initials.toUpperCase(); + } + + const svg = `\n\n \n ${ initials }\n \n`; + res.write(svg); + res.end(); + + return; + } + + const reqModifiedHeader = req.headers['if-modified-since']; + if (reqModifiedHeader) { + if (reqModifiedHeader === (file.uploadDate && file.uploadDate.toUTCString())) { + res.setHeader('Last-Modified', reqModifiedHeader); + res.writeHead(304); + res.end(); + return; + } + } + + res.setHeader('Cache-Control', 'public, max-age=0'); + res.setHeader('Expires', '-1'); + res.setHeader('Last-Modified', (file.uploadDate && file.uploadDate.toUTCString()) || new Date().toUTCString()); + res.setHeader('Content-Length', file.length); + + if (file.contentType) { + res.setHeader('Content-Type', file.contentType); + } else { + file.readStream.once('data', function(chunk) { + const fileType = getFileType(chunk); + if (fileType) { + return res.setHeader('Content-Type', fileType.mime); + } else { + return res.setHeader('Content-Type', 'image/jpeg'); + } + }); + } + + file.readStream.pipe(res); +} From a2de9f51557ba1a522a5026a3e98e5d8e5c8c55f Mon Sep 17 00:00:00 2001 From: Diego Sampaio Date: Tue, 25 Apr 2017 09:38:03 -0300 Subject: [PATCH 02/42] Add Google Cloud Storage support to avatars --- .../client/lib/FileUploadGoogleStorage.js | 33 ++----- .../globalFileRestrictions.js | 1 + .../config/configFileUploadGoogleStorage.js | 88 ++++++++++++++----- .../client/avatar/prompt.coffee | 2 +- .../rocketchat-ui/client/lib/fileUpload.js | 2 +- 5 files changed, 73 insertions(+), 53 deletions(-) diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorage.js b/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorage.js index d1daa62ec07f..b1db2b767d89 100644 --- a/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorage.js +++ b/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorage.js @@ -1,36 +1,19 @@ /* globals FileUpload, FileUploadBase, Slingshot */ FileUpload.GoogleCloudStorage = class FileUploadGoogleCloudStorage extends FileUploadBase { - constructor(meta, file) { + constructor(directive, meta, file) { super(meta, file); - this.uploader = new Slingshot.Upload('rocketchat-uploads-gs', { rid: meta.rid }); + this.uploader = new Slingshot.Upload(directive, { rid: meta.rid }); } - start() { + start(callback) { this.uploader.send(this.file, (error, downloadUrl) => { if (this.computation) { this.computation.stop(); } if (error) { - let uploading = Session.get('uploading'); - if (!Array.isArray(uploading)) { - uploading = []; - } - - const item = _.findWhere(uploading, { id: this.id }); - - if (_.isObject(item)) { - item.error = error.error; - item.percentage = 0; - } else { - uploading.push({ - error: error.error, - percentage: 0 - }); - } - - Session.set('uploading', uploading); + return callback.call(this, error); } else { const file = _.pick(this.meta, 'type', 'size', 'name', 'identify', 'description'); file._id = downloadUrl.substr(downloadUrl.lastIndexOf('/') + 1); @@ -38,13 +21,7 @@ FileUpload.GoogleCloudStorage = class FileUploadGoogleCloudStorage extends FileU Meteor.call('sendFileMessage', this.meta.rid, 'googleCloudStorage', file, () => { Meteor.setTimeout(() => { - const uploading = Session.get('uploading'); - if (uploading !== null) { - const item = _.findWhere(uploading, { - id: this.id - }); - return Session.set('uploading', _.without(uploading, item)); - } + callback.call(this, null, file); }, 2000); }); } diff --git a/packages/rocketchat-file-upload/globalFileRestrictions.js b/packages/rocketchat-file-upload/globalFileRestrictions.js index 3c52390c4341..8e2d5086dee2 100644 --- a/packages/rocketchat-file-upload/globalFileRestrictions.js +++ b/packages/rocketchat-file-upload/globalFileRestrictions.js @@ -28,3 +28,4 @@ const slingShotConfig = { Slingshot.fileRestrictions('rocketchat-avatars', slingShotConfig); Slingshot.fileRestrictions('rocketchat-uploads', slingShotConfig); Slingshot.fileRestrictions('rocketchat-uploads-gs', slingShotConfig); +Slingshot.fileRestrictions('rocketchat-avatars-gs', slingShotConfig); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js index 9e8d5219ba60..b7ae3dfa455a 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js @@ -43,6 +43,25 @@ function generateDeleteUrl({ file }) { return `https://${ file.googleCloudStorage.bucket }.storage.googleapis.com/${ encodeURIComponent(parts.path) }?GoogleAccessId=${ parts.accessId }&Expires=${ expires }&Signature=${ encodeURIComponent(signature) }`; } +function createDirective(directiveName, { key, bucket, accessId, secret }) { + if (Slingshot._directives[directiveName]) { + delete Slingshot._directives[directiveName]; + } + + const config = { + bucket, + GoogleAccessId: accessId, + GoogleSecretKey: secret, + key + }; + + try { + Slingshot.createDirective(directiveName, Slingshot.GoogleCloud, config); + } catch (e) { + console.error('Error configuring GoogleCloudStorage ->', e.message); + } +} + FileUpload.addHandler('googleCloudStorage', { get(file, req, res) { const fileUrl = generateGetURL({ file }); @@ -71,7 +90,43 @@ FileUpload.addHandler('googleCloudStorage', { }); const createGoogleStorageDirective = _.debounce(() => { - const directiveName = 'rocketchat-uploads-gs'; + const directives = [ + { + name: 'rocketchat-uploads-gs', + key: function _googleCloudStorageKey(file, metaContext) { + const path = `${ RocketChat.settings.get('uniqueID') }/${ metaContext.rid }/${ this.userId }/`; + const upload = { + rid: metaContext.rid, + googleCloudStorage: { + bucket: RocketChat.settings.get('FileUpload_GoogleStorage_Bucket'), + path + } + }; + const fileId = RocketChat.models.Uploads.insertFileInit(this.userId, 'googleCloudStorage', file, upload); + + return path + fileId; + } + }, + { + name: 'rocketchat-avatars-gs', + key(file/*, metaContext*/) { + const path = `${ RocketChat.settings.get('uniqueID') }/avatars/`; + + const user = RocketChat.models.Users.findOneById(this.userId); + + const upload = { + username: user && user.username, + googleCloudStorage: { + bucket: RocketChat.settings.get('FileUpload_GoogleStorage_Bucket'), + path + } + }; + RocketChat.models.Uploads.insertFileInitByUsername(user.username, this.userId, 'googleCloudStorage', file, upload); + + return path + user.username; + } + } + ]; const type = RocketChat.settings.get('FileUpload_Storage_Type'); const bucket = RocketChat.settings.get('FileUpload_GoogleStorage_Bucket'); @@ -79,29 +134,16 @@ const createGoogleStorageDirective = _.debounce(() => { const secret = RocketChat.settings.get('FileUpload_GoogleStorage_Secret'); if (type === 'GoogleCloudStorage' && !_.isEmpty(secret) && !_.isEmpty(accessId) && !_.isEmpty(bucket)) { - if (Slingshot._directives[directiveName]) { - delete Slingshot._directives[directiveName]; - } - - const config = { - bucket, - GoogleAccessId: accessId, - GoogleSecretKey: secret, - key: function _googleCloudStorageKey(file, metaContext) { - const path = `${ RocketChat.settings.get('uniqueID') }/${ metaContext.rid }/${ this.userId }/`; - const fileId = RocketChat.models.Uploads.insertFileInit(metaContext.rid, this.userId, 'googleCloudStorage', file, { googleCloudStorage: { bucket, path }}); - - return path + fileId; - } - }; - - try { - Slingshot.createDirective(directiveName, Slingshot.GoogleCloud, config); - } catch (e) { - SystemLogger.error('Error configuring GoogleCloudStorage ->', e.message); - } + directives.forEach((conf) => { + console.log('conf.name ->', conf.name); + createDirective(conf.name, { key: conf.key, bucket, accessId, secret }); + }); } else { - delete Slingshot._directives[directiveName]; + directives.forEach((conf) => { + if (Slingshot._directives[conf.name]) { + delete Slingshot._directives[conf.name]; + } + }); } }, 500); diff --git a/packages/rocketchat-ui-account/client/avatar/prompt.coffee b/packages/rocketchat-ui-account/client/avatar/prompt.coffee index 5b4849a1dcd2..82fa805444f9 100644 --- a/packages/rocketchat-ui-account/client/avatar/prompt.coffee +++ b/packages/rocketchat-ui-account/client/avatar/prompt.coffee @@ -75,7 +75,7 @@ Template.avatarPrompt.events type: files[0].type # description: document.getElementById('file-description').value - upload = fileUploadHandler 'rocketchat-avatars', record, files[0] + upload = fileUploadHandler 'rocketchat-avatars-gs', record, files[0] # upload.onProgress = (progress) -> # console.log 'progress ->', progress diff --git a/packages/rocketchat-ui/client/lib/fileUpload.js b/packages/rocketchat-ui/client/lib/fileUpload.js index ad91b1fc40e8..d7ef1590a437 100644 --- a/packages/rocketchat-ui/client/lib/fileUpload.js +++ b/packages/rocketchat-ui/client/lib/fileUpload.js @@ -145,7 +145,7 @@ fileUpload = function(filesToUpload) { description: document.getElementById('file-description').value }; - const upload = fileUploadHandler('rocketchat-uploads', record, file.file); + const upload = fileUploadHandler('rocketchat-uploads-gs', record, file.file); let uploading = Session.get('uploading') || []; uploading.push({ From d40d639d47af31f554a9c1cf1c340a8880ce9214 Mon Sep 17 00:00:00 2001 From: Diego Sampaio Date: Thu, 27 Apr 2017 18:14:33 -0300 Subject: [PATCH 03/42] Add GridFS support to avatars --- lib/fileUpload.js | 73 +++++++++++++++++++ .../client/lib/FileUploadGridFS.js | 36 +++------ .../client/avatar/prompt.coffee | 2 +- 3 files changed, 84 insertions(+), 27 deletions(-) diff --git a/lib/fileUpload.js b/lib/fileUpload.js index fbb006f15b7e..8f26103319de 100644 --- a/lib/fileUpload.js +++ b/lib/fileUpload.js @@ -72,6 +72,79 @@ if (UploadFS) { return true; } }); + + Meteor.fileStoreAvatar = new UploadFS.store.GridFS({ + collection: RocketChat.models.Uploads.model, + name: 'rocketchat_uploads_avatar', + collectionName: 'rocketchat_uploads', + // filter: new UploadFS.Filter({ + // onCheck: FileUpload.validateFileUpload + // }), + beforeSave() { + console.log('beforeSave ->', arguments); + }, + beforeWrite() { + console.log('beforeWrite ->', arguments); + return { + username: 'testando' + }; + }, + transformWrite(readStream, writeStream, fileId, file) { + if (RocketChatFile.enabled === false || !/^image\/.+/.test(file.type)) { + return readStream.pipe(writeStream); + } + + let stream = undefined; + + const identify = function(err, data) { + if (err) { + return stream.pipe(writeStream); + } + + file.identify = { + format: data.format, + size: data.size + }; + + if (data.Orientation && !['', 'Unknown', 'Undefined'].includes(data.Orientation)) { + RocketChatFile.gm(stream).autoOrient().stream().pipe(writeStream); + } else { + stream.pipe(writeStream); + } + }; + + stream = RocketChatFile.gm(readStream).identify(identify).stream(); + }, + + onRead(fileId, file, req, res) { + if (RocketChat.settings.get('FileUpload_ProtectFiles')) { + let uid; + let token; + + if (req && req.headers && req.headers.cookie) { + const rawCookies = req.headers.cookie; + + if (rawCookies) { + uid = cookie.get('rc_uid', rawCookies) ; + token = cookie.get('rc_token', rawCookies); + } + } + + if (!uid) { + uid = req.query.rc_uid; + token = req.query.rc_token; + } + + if (!uid || !token || !RocketChat.models.Users.findOneByIdAndLoginToken(uid, token)) { + res.writeHead(403); + return false; + } + } + + res.setHeader('content-disposition', `attachment; filename="${ encodeURIComponent(file.name) }"`); + return true; + } + }); }; Meteor.startup(function() { diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadGridFS.js b/packages/rocketchat-file-upload/client/lib/FileUploadGridFS.js index ac14bb7fafd0..a952090d87d6 100644 --- a/packages/rocketchat-file-upload/client/lib/FileUploadGridFS.js +++ b/packages/rocketchat-file-upload/client/lib/FileUploadGridFS.js @@ -1,38 +1,24 @@ /* globals FileUploadBase, UploadFS, FileUpload:true */ FileUpload.GridFS = class FileUploadGridFS extends FileUploadBase { - constructor(meta, file) { + constructor(directive, meta, file) { super(meta, file); + this.store = directive === 'avatar' ? Meteor.fileStoreAvatar : Meteor.fileStore; + } + + start(callback) { this.handler = new UploadFS.Uploader({ - store: Meteor.fileStore, - data: file, - file: meta, + store: this.store, + data: this.file, + file: this.meta, onError: (err) => { - const uploading = Session.get('uploading'); - if (uploading != null) { - const item = _.findWhere(uploading, { - id: this.id - }); - if (item != null) { - item.error = err.reason; - item.percentage = 0; - } - return Session.set('uploading', uploading); - } + return callback(err); }, onComplete: (fileData) => { const file = _.pick(fileData, '_id', 'type', 'size', 'name', 'identify', 'description'); - file.url = fileData.url.replace(Meteor.absoluteUrl(), '/'); - Meteor.call('sendFileMessage', this.meta.rid, null, file, () => { Meteor.setTimeout(() => { - const uploading = Session.get('uploading'); - if (uploading != null) { - const item = _.findWhere(uploading, { - id: this.id - }); - return Session.set('uploading', _.without(uploading, item)); - } + return callback(null, file); }, 2000); }); } @@ -41,9 +27,7 @@ FileUpload.GridFS = class FileUploadGridFS extends FileUploadBase { this.handler.onProgress = (file, progress) => { this.onProgress(progress); }; - } - start() { return this.handler.start(); } diff --git a/packages/rocketchat-ui-account/client/avatar/prompt.coffee b/packages/rocketchat-ui-account/client/avatar/prompt.coffee index 82fa805444f9..17f255c3cbd0 100644 --- a/packages/rocketchat-ui-account/client/avatar/prompt.coffee +++ b/packages/rocketchat-ui-account/client/avatar/prompt.coffee @@ -75,7 +75,7 @@ Template.avatarPrompt.events type: files[0].type # description: document.getElementById('file-description').value - upload = fileUploadHandler 'rocketchat-avatars-gs', record, files[0] + upload = fileUploadHandler 'avatar', record, files[0] # upload.onProgress = (progress) -> # console.log 'progress ->', progress From 4785fd479a3dfb87625c8cedc3b9dfabc7b9bfc6 Mon Sep 17 00:00:00 2001 From: Diego Sampaio Date: Fri, 28 Apr 2017 10:20:56 -0300 Subject: [PATCH 04/42] Better handling for GridFS avatars --- lib/fileUpload.js | 21 +- .../client/lib/FileUploadGridFS.js | 4 + .../server/config/configFileUploadAmazonS3.js | 2 +- .../config/configFileUploadGoogleStorage.js | 2 +- .../server/config/configFileUploadGridFS.js | 28 +++ .../server/models/Uploads.coffee | 23 +- server/methods/setAvatarFromService.js | 33 ++- server/startup/avatar.js | 219 +++++------------- 8 files changed, 153 insertions(+), 179 deletions(-) diff --git a/lib/fileUpload.js b/lib/fileUpload.js index 8f26103319de..f466da9ceb2d 100644 --- a/lib/fileUpload.js +++ b/lib/fileUpload.js @@ -80,15 +80,6 @@ if (UploadFS) { // filter: new UploadFS.Filter({ // onCheck: FileUpload.validateFileUpload // }), - beforeSave() { - console.log('beforeSave ->', arguments); - }, - beforeWrite() { - console.log('beforeWrite ->', arguments); - return { - username: 'testando' - }; - }, transformWrite(readStream, writeStream, fileId, file) { if (RocketChatFile.enabled === false || !/^image\/.+/.test(file.type)) { return readStream.pipe(writeStream); @@ -116,6 +107,18 @@ if (UploadFS) { stream = RocketChatFile.gm(readStream).identify(identify).stream(); }, + onFinishUpload(file) { + // update file record to match user's username + const user = RocketChat.models.Users.findOneById(file.userId); + const oldAvatar = RocketChat.models.Uploads.findOneByName(`${ user.username }.avatar`); + if (oldAvatar) { + Meteor.fileStoreAvatar.delete(oldAvatar._id); + RocketChat.models.Uploads.deleteFile(oldAvatar._id); + } + RocketChat.models.Uploads.updateFileNameById(file._id, `${ user.username }.avatar`); + // console.log('upload finished ->', file); + }, + onRead(fileId, file, req, res) { if (RocketChat.settings.get('FileUpload_ProtectFiles')) { let uid; diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadGridFS.js b/packages/rocketchat-file-upload/client/lib/FileUploadGridFS.js index a952090d87d6..d5236b1c971c 100644 --- a/packages/rocketchat-file-upload/client/lib/FileUploadGridFS.js +++ b/packages/rocketchat-file-upload/client/lib/FileUploadGridFS.js @@ -2,6 +2,7 @@ FileUpload.GridFS = class FileUploadGridFS extends FileUploadBase { constructor(directive, meta, file) { super(meta, file); + this.directive = directive; this.store = directive === 'avatar' ? Meteor.fileStoreAvatar : Meteor.fileStore; } @@ -15,6 +16,9 @@ FileUpload.GridFS = class FileUploadGridFS extends FileUploadBase { }, onComplete: (fileData) => { const file = _.pick(fileData, '_id', 'type', 'size', 'name', 'identify', 'description'); + if (this.directive === 'avatar') { + return callback(null, file); + } file.url = fileData.url.replace(Meteor.absoluteUrl(), '/'); Meteor.call('sendFileMessage', this.meta.rid, null, file, () => { Meteor.setTimeout(() => { diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js index 284edb19921a..95d1dcd84347 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js @@ -105,7 +105,7 @@ const configureSlingshot = _.debounce(() => { path } }; - RocketChat.models.Uploads.insertFileInitByUsername(user.username, this.userId, 's3', file, upload); + RocketChat.models.Uploads.insertAvatarFileInit(user.username, this.userId, 's3', file, upload); return path + user.username; } diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js index b7ae3dfa455a..b2d1c6b9a826 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js @@ -121,7 +121,7 @@ const createGoogleStorageDirective = _.debounce(() => { path } }; - RocketChat.models.Uploads.insertFileInitByUsername(user.username, this.userId, 'googleCloudStorage', file, upload); + RocketChat.models.Uploads.insertAvatarFileInit(user.username, this.userId, 'googleCloudStorage', file, upload); return path + user.username; } diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js b/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js index 21ac212db8ed..fad40155182d 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js @@ -139,3 +139,31 @@ FileUpload.addHandler('rocketchat_uploads', { return Meteor.fileStore.delete(file._id); } }); + +FileUpload.addHandler('rocketchat_uploads_avatar', { + get(file, req, res) { + const reqModifiedHeader = req.headers['if-modified-since']; + if (reqModifiedHeader) { + if (reqModifiedHeader === (file.uploadedAt && file.uploadedAt.toUTCString())) { + res.setHeader('Last-Modified', reqModifiedHeader); + res.writeHead(304); + res.end(); + return; + } + } + + file = FileUpload.addExtensionTo(file); + const headers = { + 'Cache-Control': 'public, max-age=0', + 'Expires': '-1', + 'Content-Disposition': 'inline', + 'Last-Modified': file.uploadedAt.toUTCString(), + 'Content-Type': file.type, + 'Content-Length': file.size + }; + return readFromGridFS(file.store, file._id, file, headers, req, res); + }, + delete(file) { + return Meteor.fileStore.delete(file._id); + } +}); diff --git a/packages/rocketchat-lib/server/models/Uploads.coffee b/packages/rocketchat-lib/server/models/Uploads.coffee index 2604403bbe6c..6fc165a190c1 100644 --- a/packages/rocketchat-lib/server/models/Uploads.coffee +++ b/packages/rocketchat-lib/server/models/Uploads.coffee @@ -48,9 +48,9 @@ RocketChat.models.Uploads = new class extends RocketChat.models._Base return file - insertFileInitByUsername: (username, userId, store, file, extra) -> + insertAvatarFileInit: (name, userId, store, file, extra) -> fileData = - _id: username + name: "#{name}.avatar" userId: userId store: store complete: false @@ -88,12 +88,13 @@ RocketChat.models.Uploads = new class extends RocketChat.models._Base return result - updateFileCompleteByUsername: (username, userId, url) -> - if not username + # @TODO deprecated + updateFileCompleteByNameAndUserId: (name, userId, url) -> + if not name return filter = - username: username + name: name userId: userId update = @@ -110,11 +111,19 @@ RocketChat.models.Uploads = new class extends RocketChat.models._Base return result - findOneByUsername: (username) -> - return @findOne username: username + findOneByName: (name) -> + return @findOne name: name deleteFile: (fileId) -> if @model.direct?.remove? return @model.direct.remove { _id: fileId } else return @remove { _id: fileId } + + updateFileNameById: (fileId, name) -> + filter = _id: fileId + update = $set: name: name + if @model.direct?.update? + return @model.direct.update filter, update + else + return @update filter, update diff --git a/server/methods/setAvatarFromService.js b/server/methods/setAvatarFromService.js index bdc0768552c3..8781e1c45022 100644 --- a/server/methods/setAvatarFromService.js +++ b/server/methods/setAvatarFromService.js @@ -22,10 +22,37 @@ Meteor.methods({ }, saveAvatarFile(file) { + check(file, Match.ObjectIncluding({ + _id: String + })); + + if (!Meteor.userId()) { + throw new Meteor.Error('error-invalid-user', 'Invalid user', { + method: 'setAvatarFromService' + }); + } + + if (!RocketChat.settings.get('Accounts_AllowUserAvatarChange')) { + throw new Meteor.Error('error-not-allowed', 'Not allowed', { + method: 'setAvatarFromService' + }); + } + const user = RocketChat.models.Users.findOneById(Meteor.userId()); - // console.log('user ->', user); - // console.log('file ->', file); - RocketChat.models.Uploads.updateFileCompleteByUsername(user.username, Meteor.userId(), file.url); + const fileSaved = RocketChat.models.Uploads.findOneById(file._id); + + if (fileSaved.userId !== user._id) { + // this file is not user's avatar + throw new Meteor.Error('invalid-avatar'); + } + + // just returns if file already complete (case for GridFS) + if (fileSaved.complete) { + return true; + } + + RocketChat.models.Uploads.updateFileCompleteByNameAndUserId(`${ user.username }.avatar`, Meteor.userId(), file.url); + return true; } }); diff --git a/server/startup/avatar.js b/server/startup/avatar.js index 634effe061b2..cf31fcc2366b 100644 --- a/server/startup/avatar.js +++ b/server/startup/avatar.js @@ -1,5 +1,4 @@ -import getFileType from 'file-type'; - +/* globals FileUpload */ Meteor.startup(function() { let storeType = 'GridFS'; @@ -47,183 +46,87 @@ Meteor.startup(function() { return; } - return getNewAvatar(params, req, res); - })); -}); - - -function getNewAvatar(params, req, res) { - const match = /^\/([^?]*)/.exec(req.url); - - if (match[1]) { - let username = decodeURIComponent(match[1]); - const file = RocketChat.models.Uploads.findOneByUsername(username); - - if (file) { - res.setHeader('Content-Security-Policy', 'default-src \'none\''); + const match = /^\/([^?]*)/.exec(req.url); - return FileUpload.get(file, req, res); - } else { - res.setHeader('Content-Type', 'image/svg+xml'); - res.setHeader('Cache-Control', 'public, max-age=0'); - res.setHeader('Expires', '-1'); - res.setHeader('Last-Modified', 'Thu, 01 Jan 2015 00:00:00 GMT'); + if (match[1]) { + let username = decodeURIComponent(match[1]); + let file; - const reqModifiedHeader = req.headers['if-modified-since']; - - if (reqModifiedHeader) { - if (reqModifiedHeader === 'Thu, 01 Jan 2015 00:00:00 GMT') { - res.writeHead(304); - res.end(); - return; - } - } - - const colors = ['#F44336', '#E91E63', '#9C27B0', '#673AB7', '#3F51B5', '#2196F3', '#03A9F4', '#00BCD4', '#009688', '#4CAF50', '#8BC34A', '#CDDC39', '#FFC107', '#FF9800', '#FF5722', '#795548', '#9E9E9E', '#607D8B']; - - if (RocketChat.settings.get('UI_Use_Name_Avatar')) { - const user = RocketChat.models.Users.findOneByUsername(username, { - fields: { - name: 1 + if (username[0] !== '@') { + if (Meteor.settings && Meteor.settings.public && Meteor.settings.public.sandstorm) { + const user = RocketChat.models.Users.findOneByUsername(username); + if (user && user.services && user.services.sandstorm && user.services.sandstorm.picture) { + res.setHeader('Location', user.services.sandstorm.picture); + res.writeHead(302); + res.end(); + return; } - }); - - if (user && user.name) { - username = user.name; } + file = RocketChat.models.Uploads.findOneByName(`${ username }.avatar`); } - let color = ''; - let initials = ''; + if (file) { + res.setHeader('Content-Security-Policy', 'default-src \'none\''); - if (username === '?') { - color = '#000'; - initials = username; + return FileUpload.get(file, req, res); } else { - const position = username.length % colors.length; - - color = colors[position]; - username = username.replace(/[^A-Za-z0-9]/g, '.').replace(/\.+/g, '.').replace(/(^\.)|(\.$)/g, ''); + res.setHeader('Content-Type', 'image/svg+xml'); + res.setHeader('Cache-Control', 'public, max-age=0'); + res.setHeader('Expires', '-1'); + res.setHeader('Last-Modified', 'Thu, 01 Jan 2015 00:00:00 GMT'); + + const reqModifiedHeader = req.headers['if-modified-since']; + + if (reqModifiedHeader) { + if (reqModifiedHeader === 'Thu, 01 Jan 2015 00:00:00 GMT') { + res.writeHead(304); + res.end(); + return; + } + } - const usernameParts = username.split('.'); + const colors = ['#F44336', '#E91E63', '#9C27B0', '#673AB7', '#3F51B5', '#2196F3', '#03A9F4', '#00BCD4', '#009688', '#4CAF50', '#8BC34A', '#CDDC39', '#FFC107', '#FF9800', '#FF5722', '#795548', '#9E9E9E', '#607D8B']; - initials = usernameParts.length > 1 ? _.first(usernameParts)[0] + _.last(usernameParts)[0] : username.replace(/[^A-Za-z0-9]/g, '').substr(0, 2); - initials = initials.toUpperCase(); - } + if (RocketChat.settings.get('UI_Use_Name_Avatar')) { + const user = RocketChat.models.Users.findOneByUsername(username, { + fields: { + name: 1 + } + }); - const svg = `\n\n \n ${ initials }\n \n`; - res.write(svg); - res.end(); + if (user && user.name) { + username = user.name; + } + } - return; - } - } + let color = ''; + let initials = ''; - res.writeHead(404); - res.end(); - return; -} - -function getOldAvatar(params, req, res) { - let username = params.username.replace(/\.jpg$/, '').replace(/^@/, ''); - let file; - if (params.username[0] !== '@') { - if (Meteor.settings && Meteor.settings.public && Meteor.settings.public.sandstorm) { - const user = RocketChat.models.Users.findOneByUsername(username); - if (user && user.services && user.services.sandstorm && user.services.sandstorm.picture) { - res.setHeader('Location', user.services.sandstorm.picture); - res.writeHead(302); - res.end(); - return; - } - } + if (username === '?') { + color = '#000'; + initials = username; + } else { + const position = username.length % colors.length; - file = RocketChatFileAvatarInstance.getFileWithReadStream(encodeURIComponent(`${ username }.jpg`)); - } + color = colors[position]; + username = username.replace(/[^A-Za-z0-9]/g, '.').replace(/\.+/g, '.').replace(/(^\.)|(\.$)/g, ''); - res.setHeader('Content-Disposition', 'inline'); - if (!file) { - res.setHeader('Content-Type', 'image/svg+xml'); - res.setHeader('Cache-Control', 'public, max-age=0'); - res.setHeader('Expires', '-1'); - res.setHeader('Last-Modified', 'Thu, 01 Jan 2015 00:00:00 GMT'); + const usernameParts = username.split('.'); - const reqModifiedHeader = req.headers['if-modified-since']; + initials = usernameParts.length > 1 ? _.first(usernameParts)[0] + _.last(usernameParts)[0] : username.replace(/[^A-Za-z0-9]/g, '').substr(0, 2); + initials = initials.toUpperCase(); + } - if (reqModifiedHeader) { - if (reqModifiedHeader === 'Thu, 01 Jan 2015 00:00:00 GMT') { - res.writeHead(304); + const svg = `\n\n \n ${ initials }\n \n`; + res.write(svg); res.end(); - return; - } - } - - const colors = ['#F44336', '#E91E63', '#9C27B0', '#673AB7', '#3F51B5', '#2196F3', '#03A9F4', '#00BCD4', '#009688', '#4CAF50', '#8BC34A', '#CDDC39', '#FFC107', '#FF9800', '#FF5722', '#795548', '#9E9E9E', '#607D8B']; - - if (RocketChat.settings.get('UI_Use_Name_Avatar')) { - const user = RocketChat.models.Users.findOneByUsername(username, { - fields: { - name: 1 - } - }); - if (user && user.name) { - username = user.name; + return; } } - let color = ''; - let initials = ''; - - if (username === '?') { - color = '#000'; - initials = username; - } else { - const position = username.length % colors.length; - - color = colors[position]; - username = username.replace(/[^A-Za-z0-9]/g, '.').replace(/\.+/g, '.').replace(/(^\.)|(\.$)/g, ''); - - const usernameParts = username.split('.'); - - initials = usernameParts.length > 1 ? _.first(usernameParts)[0] + _.last(usernameParts)[0] : username.replace(/[^A-Za-z0-9]/g, '').substr(0, 2); - initials = initials.toUpperCase(); - } - - const svg = `\n\n \n ${ initials }\n \n`; - res.write(svg); + res.writeHead(404); res.end(); - return; - } - - const reqModifiedHeader = req.headers['if-modified-since']; - if (reqModifiedHeader) { - if (reqModifiedHeader === (file.uploadDate && file.uploadDate.toUTCString())) { - res.setHeader('Last-Modified', reqModifiedHeader); - res.writeHead(304); - res.end(); - return; - } - } - - res.setHeader('Cache-Control', 'public, max-age=0'); - res.setHeader('Expires', '-1'); - res.setHeader('Last-Modified', (file.uploadDate && file.uploadDate.toUTCString()) || new Date().toUTCString()); - res.setHeader('Content-Length', file.length); - - if (file.contentType) { - res.setHeader('Content-Type', file.contentType); - } else { - file.readStream.once('data', function(chunk) { - const fileType = getFileType(chunk); - if (fileType) { - return res.setHeader('Content-Type', fileType.mime); - } else { - return res.setHeader('Content-Type', 'image/jpeg'); - } - }); - } - - file.readStream.pipe(res); -} + })); +}); From b7fc7b68daa0b0c9bc81778f5aca60165a5c389d Mon Sep 17 00:00:00 2001 From: Diego Sampaio Date: Fri, 28 Apr 2017 17:46:05 -0300 Subject: [PATCH 05/42] Use new collection for avatars only --- lib/fileUpload.js | 10 +- .../client/lib/FileUploadAmazonS3.js | 11 +- .../client/lib/FileUploadGoogleStorage.js | 11 +- .../server/config/configFileUploadAmazonS3.js | 3 +- .../config/configFileUploadGoogleStorage.js | 3 +- .../rocketchat-lib/client/models/Avatars.js | 6 + packages/rocketchat-lib/package.js | 2 + .../rocketchat-lib/server/models/Avatars.js | 105 ++++++++++++++++++ .../server/models/Uploads.coffee | 51 --------- server/methods/setAvatarFromService.js | 9 +- server/startup/avatar.js | 2 +- 11 files changed, 150 insertions(+), 63 deletions(-) create mode 100644 packages/rocketchat-lib/client/models/Avatars.js create mode 100644 packages/rocketchat-lib/server/models/Avatars.js diff --git a/lib/fileUpload.js b/lib/fileUpload.js index f466da9ceb2d..93ae93979284 100644 --- a/lib/fileUpload.js +++ b/lib/fileUpload.js @@ -74,9 +74,9 @@ if (UploadFS) { }); Meteor.fileStoreAvatar = new UploadFS.store.GridFS({ - collection: RocketChat.models.Uploads.model, + collection: RocketChat.models.Avatars.model, name: 'rocketchat_uploads_avatar', - collectionName: 'rocketchat_uploads', + collectionName: 'rocketchat_avatars', // filter: new UploadFS.Filter({ // onCheck: FileUpload.validateFileUpload // }), @@ -110,12 +110,12 @@ if (UploadFS) { onFinishUpload(file) { // update file record to match user's username const user = RocketChat.models.Users.findOneById(file.userId); - const oldAvatar = RocketChat.models.Uploads.findOneByName(`${ user.username }.avatar`); + const oldAvatar = RocketChat.models.Avatars.findOneByName(`${ user.username }.avatar`); if (oldAvatar) { Meteor.fileStoreAvatar.delete(oldAvatar._id); - RocketChat.models.Uploads.deleteFile(oldAvatar._id); + RocketChat.models.Avatars.deleteFile(oldAvatar._id); } - RocketChat.models.Uploads.updateFileNameById(file._id, `${ user.username }.avatar`); + RocketChat.models.Avatars.updateFileNameById(file._id, `${ user.username }.avatar`); // console.log('upload finished ->', file); }, diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js b/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js index df9fa58e3f56..7d8250b7ddfa 100644 --- a/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js +++ b/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js @@ -3,7 +3,12 @@ FileUpload.AmazonS3 = class FileUploadAmazonS3 extends FileUploadBase { constructor(directive, meta, file) { super(meta, file); - this.uploader = new Slingshot.Upload(directive, meta); + this.directive = directive; + const directives = { + 'upload': 'rocketchat-uploads', + 'avatar': 'rocketchat-avatars' + }; + this.uploader = new Slingshot.Upload(directives[directive], meta); } start(callback) { @@ -19,6 +24,10 @@ FileUpload.AmazonS3 = class FileUploadAmazonS3 extends FileUploadBase { file._id = downloadUrl.substr(downloadUrl.lastIndexOf('/') + 1); file.url = downloadUrl; + if (this.directive === 'avatar') { + return callback(null, file); + } + Meteor.call('sendFileMessage', this.meta.rid, 's3', file, () => { Meteor.setTimeout(() => { callback.call(this, null, file); diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorage.js b/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorage.js index b1db2b767d89..1ac7279e4bf6 100644 --- a/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorage.js +++ b/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorage.js @@ -3,7 +3,12 @@ FileUpload.GoogleCloudStorage = class FileUploadGoogleCloudStorage extends FileUploadBase { constructor(directive, meta, file) { super(meta, file); - this.uploader = new Slingshot.Upload(directive, { rid: meta.rid }); + this.directive = directive; + const directives = { + 'upload': 'rocketchat-uploads-gs', + 'avatar': 'rocketchat-avatars-gs' + }; + this.uploader = new Slingshot.Upload(directives[directive], { rid: meta.rid }); } start(callback) { @@ -19,6 +24,10 @@ FileUpload.GoogleCloudStorage = class FileUploadGoogleCloudStorage extends FileU file._id = downloadUrl.substr(downloadUrl.lastIndexOf('/') + 1); file.url = downloadUrl; + if (this.directive === 'avatar') { + return callback(null, file); + } + Meteor.call('sendFileMessage', this.meta.rid, 'googleCloudStorage', file, () => { Meteor.setTimeout(() => { callback.call(this, null, file); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js index 95d1dcd84347..a1844ba22e06 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js @@ -105,7 +105,8 @@ const configureSlingshot = _.debounce(() => { path } }; - RocketChat.models.Uploads.insertAvatarFileInit(user.username, this.userId, 's3', file, upload); + delete file.name; + RocketChat.models.Avatars.insertAvatarFileInit(user.username, this.userId, 's3', file, upload); return path + user.username; } diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js index b2d1c6b9a826..e7a2b7b4fc8e 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js @@ -121,7 +121,8 @@ const createGoogleStorageDirective = _.debounce(() => { path } }; - RocketChat.models.Uploads.insertAvatarFileInit(user.username, this.userId, 'googleCloudStorage', file, upload); + delete file.name; + RocketChat.models.Avatars.insertAvatarFileInit(user.username, this.userId, 'googleCloudStorage', file, upload); return path + user.username; } diff --git a/packages/rocketchat-lib/client/models/Avatars.js b/packages/rocketchat-lib/client/models/Avatars.js new file mode 100644 index 000000000000..bd8804e746c8 --- /dev/null +++ b/packages/rocketchat-lib/client/models/Avatars.js @@ -0,0 +1,6 @@ +RocketChat.models.Avatars = new class extends RocketChat.models._Base { + constructor() { + super(); + this._initModel('avatars'); + } +}; diff --git a/packages/rocketchat-lib/package.js b/packages/rocketchat-lib/package.js index 8902fce8f259..9ec626890d5e 100644 --- a/packages/rocketchat-lib/package.js +++ b/packages/rocketchat-lib/package.js @@ -104,6 +104,7 @@ Package.onUse(function(api) { // SERVER MODELS api.addFiles('server/models/_Base.js', 'server'); + api.addFiles('server/models/Avatars.js', 'server'); api.addFiles('server/models/Messages.coffee', 'server'); api.addFiles('server/models/Reports.js', 'server'); api.addFiles('server/models/Rooms.coffee', 'server'); @@ -195,6 +196,7 @@ Package.onUse(function(api) { // CLIENT MODELS api.addFiles('client/models/_Base.coffee', 'client'); + api.addFiles('client/models/Avatars.js', 'client'); api.addFiles('client/models/Uploads.coffee', 'client'); // CLIENT VIEWS diff --git a/packages/rocketchat-lib/server/models/Avatars.js b/packages/rocketchat-lib/server/models/Avatars.js new file mode 100644 index 000000000000..43c7b6be6af6 --- /dev/null +++ b/packages/rocketchat-lib/server/models/Avatars.js @@ -0,0 +1,105 @@ +RocketChat.models.Avatars = new class extends RocketChat.models._Base { + constructor() { + super('avatars'); + + this.tryEnsureIndex({ name: 1 }); + } + + insertAvatarFileInit(name, userId, store, file, extra) { + const fileData = { + _id: name, + name, + userId, + store, + complete: false, + uploading: true, + progress: 0, + extension: s.strRightBack(file.name, '.'), + uploadedAt: new Date() + }; + + _.extend(fileData, file, extra); + + return this.insertOrUpsert(fileData); + } + + updateFileComplete(fileId, userId, file) { + if (!fileId) { + return; + } + + const filter = { + _id: fileId, + userId + }; + + const update = { + $set: { + complete: true, + uploading: false, + progress: 1 + } + }; + + update.$set = _.extend(file, update.$set); + + if (this.model.direct && this.model.direct.update) { + return this.model.direct.update(filter, update); + } else { + return this.update(filter, update); + } + } + + findOneByName(name) { + return this.findOne({ name }); + } + + updateFileNameById(fileId, name) { + const filter = { _id: fileId }; + const update = { + $set: { + name + } + }; + if (this.model.direct && this.model.direct.update) { + return this.model.direct.update(filter, update); + } else { + return this.update(filter, update); + } + } + + // @TODO deprecated + updateFileCompleteByNameAndUserId(name, userId, url) { + if (!name) { + return; + } + + const filter = { + name, + userId + }; + + const update = { + $set: { + complete: true, + uploading: false, + progress: 1, + url + } + }; + + if (this.model.direct && this.model.direct.update) { + return this.model.direct.update(filter, update); + } else { + return this.update(filter, update); + } + } + + deleteFile(fileId) { + if (this.model.direct && this.model.direct.remove) { + return this.model.direct.remove({ _id: fileId }); + } else { + return this.remove({ _id: fileId }); + } + } +}; diff --git a/packages/rocketchat-lib/server/models/Uploads.coffee b/packages/rocketchat-lib/server/models/Uploads.coffee index 6fc165a190c1..ff5edf5fd8d3 100644 --- a/packages/rocketchat-lib/server/models/Uploads.coffee +++ b/packages/rocketchat-lib/server/models/Uploads.coffee @@ -48,23 +48,6 @@ RocketChat.models.Uploads = new class extends RocketChat.models._Base return file - insertAvatarFileInit: (name, userId, store, file, extra) -> - fileData = - name: "#{name}.avatar" - userId: userId - store: store - complete: false - uploading: true - progress: 0 - extension: s.strRightBack(file.name, '.') - uploadedAt: new Date() - - _.extend(fileData, file, extra); - - file = @insertOrUpsert fileData - - return file - updateFileComplete: (fileId, userId, file) -> if not fileId return @@ -88,42 +71,8 @@ RocketChat.models.Uploads = new class extends RocketChat.models._Base return result - # @TODO deprecated - updateFileCompleteByNameAndUserId: (name, userId, url) -> - if not name - return - - filter = - name: name - userId: userId - - update = - $set: - complete: true - uploading: false - progress: 1 - url: url - - if @model.direct?.update? - result = @model.direct.update filter, update - else - result = @update filter, update - - return result - - findOneByName: (name) -> - return @findOne name: name - deleteFile: (fileId) -> if @model.direct?.remove? return @model.direct.remove { _id: fileId } else return @remove { _id: fileId } - - updateFileNameById: (fileId, name) -> - filter = _id: fileId - update = $set: name: name - if @model.direct?.update? - return @model.direct.update filter, update - else - return @update filter, update diff --git a/server/methods/setAvatarFromService.js b/server/methods/setAvatarFromService.js index 8781e1c45022..6f974d1a35cc 100644 --- a/server/methods/setAvatarFromService.js +++ b/server/methods/setAvatarFromService.js @@ -39,7 +39,11 @@ Meteor.methods({ } const user = RocketChat.models.Users.findOneById(Meteor.userId()); - const fileSaved = RocketChat.models.Uploads.findOneById(file._id); + const fileSaved = RocketChat.models.Avatars.findOneById(file._id); + + if (!fileSaved) { + return; + } if (fileSaved.userId !== user._id) { // this file is not user's avatar @@ -51,7 +55,8 @@ Meteor.methods({ return true; } - RocketChat.models.Uploads.updateFileCompleteByNameAndUserId(`${ user.username }.avatar`, Meteor.userId(), file.url); + RocketChat.models.Avatars.updateFileCompleteByNameAndUserId(user.username, user._id, file.url); + return true; } }); diff --git a/server/startup/avatar.js b/server/startup/avatar.js index cf31fcc2366b..ae69381aff5e 100644 --- a/server/startup/avatar.js +++ b/server/startup/avatar.js @@ -62,7 +62,7 @@ Meteor.startup(function() { return; } } - file = RocketChat.models.Uploads.findOneByName(`${ username }.avatar`); + file = RocketChat.models.Avatars.findOneByName(username); } if (file) { From ce812e804b46a290836eba9777b3f0bd42804fb6 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Mon, 1 May 2017 12:17:15 -0300 Subject: [PATCH 06/42] Delete converted file --- .../client/lib/fileUpload.coffee | 187 ------------------ 1 file changed, 187 deletions(-) delete mode 100644 packages/rocketchat-ui/client/lib/fileUpload.coffee diff --git a/packages/rocketchat-ui/client/lib/fileUpload.coffee b/packages/rocketchat-ui/client/lib/fileUpload.coffee deleted file mode 100644 index 55507fe06189..000000000000 --- a/packages/rocketchat-ui/client/lib/fileUpload.coffee +++ /dev/null @@ -1,187 +0,0 @@ -readAsDataURL = (file, callback) -> - reader = new FileReader() - reader.onload = (ev) -> - callback ev.target.result, file - - reader.readAsDataURL file - -getUploadPreview = (file, callback) -> - # If greater then 10MB don't try and show a preview - if file.file.size > 10 * 1000000 - callback(file, null) - else - if file.file.type.indexOf('audio') > -1 or file.file.type.indexOf('video') > -1 or file.file.type.indexOf('image') > -1 - file.type = file.file.type.split('/')[0] - - readAsDataURL file.file, (content) -> - callback(file, content) - else - callback(file, null) - -formatBytes = (bytes, decimals) -> - if bytes == 0 - return '0 Bytes' - - k = 1000 - dm = decimals + 1 or 3 - - sizes = [ - 'Bytes' - 'KB' - 'MB' - 'GB' - 'TB' - 'PB' - ] - - i = Math.floor(Math.log(bytes) / Math.log(k)) - - parseFloat((bytes / k ** i).toFixed(dm)) + ' ' + sizes[i] - -readAsArrayBuffer = (file, callback) -> - reader = new FileReader() - reader.onload = (ev) -> - callback ev.target.result, file - - reader.readAsArrayBuffer file - - -@fileUpload = (files) -> - roomId = Session.get('openedRoom') - files = [].concat files - - consume = -> - file = files.pop() - if not file? - swal.close() - return - - if not RocketChat.fileUploadIsValidContentType file.file.type - swal - title: t('FileUpload_MediaType_NotAccepted') - text: file.file.type || "*.#{s.strRightBack(file.file.name, '.')}" - type: 'error' - timer: 3000 - return - - if file.file.size is 0 - swal - title: t('FileUpload_File_Empty') - type: 'error' - timer: 1000 - return - - getUploadPreview file, (file, preview) -> - text = '' - - if file.type is 'audio' - text = """ -
- -
-
- - -
- """ - else if file.type is 'video' - text = """ -
- -
-
- - -
- """ - else if file.type is 'image' - text = """ -
-
-
-
- - -
- """ - else - fileSize = formatBytes(file.file.size) - - text = """ -
-
#{Handlebars._escape(file.name)} - #{fileSize}
-
-
- - -
- """ - - swal - title: t('Upload_file_question') - text: text - showCancelButton: true - closeOnConfirm: false - closeOnCancel: false - confirmButtonText: t('Send') - cancelButtonText: t('Cancel') - html: true - , (isConfirm) -> - consume() - if isConfirm isnt true - return - - record = - name: document.getElementById('file-name').value or file.name or file.file.name - size: file.file.size - type: file.file.type - rid: roomId - description: document.getElementById('file-description').value - - upload = fileUploadHandler record, file.file - - uploading = Session.get('uploading') or [] - uploading.push - id: upload.id - name: upload.getFileName() - percentage: 0 - - Session.set 'uploading', uploading - - upload.onProgress = (progress) -> - uploading = Session.get('uploading') - - item = _.findWhere(uploading, {id: upload.id}) - if item? - item.percentage = Math.round(progress * 100) or 0 - Session.set 'uploading', uploading - - upload.start() - - Tracker.autorun (c) -> - cancel = Session.get "uploading-cancel-#{upload.id}" - if cancel - upload.stop() - c.stop() - - uploading = Session.get 'uploading' - if uploading? - item = _.findWhere(uploading, {id: upload.id}) - if item? - item.percentage = 0 - Session.set 'uploading', uploading - - Meteor.setTimeout -> - uploading = Session.get 'uploading' - if uploading? - item = _.findWhere(uploading, {id: upload.id}) - Session.set 'uploading', _.without(uploading, item) - , 1000 - - consume() From 1c72057feb6d2b02c3b2b02de99a56c3a6e992c7 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Mon, 1 May 2017 12:18:05 -0300 Subject: [PATCH 07/42] Avatar: Add support to FileSystem and various fixes --- lib/fileUpload.js | 31 +-- .../client/lib/FileUploadAmazonS3.js | 11 +- .../client/lib/FileUploadFileSystem.js | 50 ++--- .../client/lib/FileUploadGoogleStorage.js | 11 +- .../client/lib/FileUploadGridFS.js | 10 +- .../config/configFileUploadFileSystem.js | 125 +++++++++--- .../server/lib/FileUpload.js | 10 + .../.npm/package/npm-shrinkwrap.json | 188 +++--------------- .../rocketchat-ui/client/lib/fileUpload.js | 27 ++- 9 files changed, 175 insertions(+), 288 deletions(-) diff --git a/lib/fileUpload.js b/lib/fileUpload.js index 93ae93979284..782015a73cd5 100644 --- a/lib/fileUpload.js +++ b/lib/fileUpload.js @@ -80,42 +80,17 @@ if (UploadFS) { // filter: new UploadFS.Filter({ // onCheck: FileUpload.validateFileUpload // }), - transformWrite(readStream, writeStream, fileId, file) { - if (RocketChatFile.enabled === false || !/^image\/.+/.test(file.type)) { - return readStream.pipe(writeStream); - } - - let stream = undefined; - - const identify = function(err, data) { - if (err) { - return stream.pipe(writeStream); - } - - file.identify = { - format: data.format, - size: data.size - }; - - if (data.Orientation && !['', 'Unknown', 'Undefined'].includes(data.Orientation)) { - RocketChatFile.gm(stream).autoOrient().stream().pipe(writeStream); - } else { - stream.pipe(writeStream); - } - }; - - stream = RocketChatFile.gm(readStream).identify(identify).stream(); - }, + transformWrite: FileUpload.avatarTransformWrite, onFinishUpload(file) { // update file record to match user's username const user = RocketChat.models.Users.findOneById(file.userId); - const oldAvatar = RocketChat.models.Avatars.findOneByName(`${ user.username }.avatar`); + const oldAvatar = RocketChat.models.Avatars.findOneByName(user.username); if (oldAvatar) { Meteor.fileStoreAvatar.delete(oldAvatar._id); RocketChat.models.Avatars.deleteFile(oldAvatar._id); } - RocketChat.models.Avatars.updateFileNameById(file._id, `${ user.username }.avatar`); + RocketChat.models.Avatars.updateFileNameById(file._id, user.username); // console.log('upload finished ->', file); }, diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js b/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js index 7d8250b7ddfa..0c6f3efd9054 100644 --- a/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js +++ b/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js @@ -3,7 +3,6 @@ FileUpload.AmazonS3 = class FileUploadAmazonS3 extends FileUploadBase { constructor(directive, meta, file) { super(meta, file); - this.directive = directive; const directives = { 'upload': 'rocketchat-uploads', 'avatar': 'rocketchat-avatars' @@ -24,15 +23,7 @@ FileUpload.AmazonS3 = class FileUploadAmazonS3 extends FileUploadBase { file._id = downloadUrl.substr(downloadUrl.lastIndexOf('/') + 1); file.url = downloadUrl; - if (this.directive === 'avatar') { - return callback(null, file); - } - - Meteor.call('sendFileMessage', this.meta.rid, 's3', file, () => { - Meteor.setTimeout(() => { - callback.call(this, null, file); - }, 2000); - }); + return callback(null, file, 's3'); } }); diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadFileSystem.js b/packages/rocketchat-file-upload/client/lib/FileUploadFileSystem.js index d19c5c03141f..025a3cf78103 100644 --- a/packages/rocketchat-file-upload/client/lib/FileUploadFileSystem.js +++ b/packages/rocketchat-file-upload/client/lib/FileUploadFileSystem.js @@ -1,4 +1,4 @@ -/* globals FileUploadBase, UploadFS, FileUpload:true, FileSystemStore:true */ +/* globals FileUploadBase, UploadFS, FileUpload:true, FileSystemStore:true, FileSystemStoreAvatar:true */ FileSystemStore = new UploadFS.store.Local({ collection: RocketChat.models.Uploads.model, @@ -8,51 +8,41 @@ FileSystemStore = new UploadFS.store.Local({ }) }); +FileSystemStoreAvatar = new UploadFS.store.Local({ + collection: RocketChat.models.Avatars.model, + name: 'fileSystemAvatar', + filter: new UploadFS.Filter({ + onCheck: FileUpload.validateFileUpload + }) +}); + FileUpload.FileSystem = class FileUploadFileSystem extends FileUploadBase { - constructor(meta, file) { + constructor(directive, meta, file) { super(meta, file); + console.log('filesystem', {directive, meta, file}); + this.store = directive === 'avatar' ? FileSystemStoreAvatar : FileSystemStore; + } + + start(callback) { this.handler = new UploadFS.Uploader({ - store: FileSystemStore, - data: file, - file: meta, + store: this.store, + data: this.file, + file: this.meta, onError: (err) => { - const uploading = Session.get('uploading'); - if (uploading != null) { - const item = _.findWhere(uploading, { - id: this.id - }); - if (item != null) { - item.error = err.reason; - item.percentage = 0; - } - return Session.set('uploading', uploading); - } + return callback(err); }, onComplete: (fileData) => { const file = _.pick(fileData, '_id', 'type', 'size', 'name', 'identify', 'description'); file.url = fileData.url.replace(Meteor.absoluteUrl(), '/'); - - Meteor.call('sendFileMessage', this.meta.rid, null, file, () => { - Meteor.setTimeout(() => { - const uploading = Session.get('uploading'); - if (uploading != null) { - const item = _.findWhere(uploading, { - id: this.id - }); - return Session.set('uploading', _.without(uploading, item)); - } - }, 2000); - }); + return callback(null, file, 'fs'); } }); this.handler.onProgress = (file, progress) => { this.onProgress(progress); }; - } - start() { return this.handler.start(); } diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorage.js b/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorage.js index 1ac7279e4bf6..b4b047c42b8b 100644 --- a/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorage.js +++ b/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorage.js @@ -3,7 +3,6 @@ FileUpload.GoogleCloudStorage = class FileUploadGoogleCloudStorage extends FileUploadBase { constructor(directive, meta, file) { super(meta, file); - this.directive = directive; const directives = { 'upload': 'rocketchat-uploads-gs', 'avatar': 'rocketchat-avatars-gs' @@ -24,15 +23,7 @@ FileUpload.GoogleCloudStorage = class FileUploadGoogleCloudStorage extends FileU file._id = downloadUrl.substr(downloadUrl.lastIndexOf('/') + 1); file.url = downloadUrl; - if (this.directive === 'avatar') { - return callback(null, file); - } - - Meteor.call('sendFileMessage', this.meta.rid, 'googleCloudStorage', file, () => { - Meteor.setTimeout(() => { - callback.call(this, null, file); - }, 2000); - }); + return callback(null, file, 'googleCloudStorage'); } }); diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadGridFS.js b/packages/rocketchat-file-upload/client/lib/FileUploadGridFS.js index d5236b1c971c..6b8c3755b99e 100644 --- a/packages/rocketchat-file-upload/client/lib/FileUploadGridFS.js +++ b/packages/rocketchat-file-upload/client/lib/FileUploadGridFS.js @@ -2,7 +2,6 @@ FileUpload.GridFS = class FileUploadGridFS extends FileUploadBase { constructor(directive, meta, file) { super(meta, file); - this.directive = directive; this.store = directive === 'avatar' ? Meteor.fileStoreAvatar : Meteor.fileStore; } @@ -16,15 +15,8 @@ FileUpload.GridFS = class FileUploadGridFS extends FileUploadBase { }, onComplete: (fileData) => { const file = _.pick(fileData, '_id', 'type', 'size', 'name', 'identify', 'description'); - if (this.directive === 'avatar') { - return callback(null, file); - } file.url = fileData.url.replace(Meteor.absoluteUrl(), '/'); - Meteor.call('sendFileMessage', this.meta.rid, null, file, () => { - Meteor.setTimeout(() => { - return callback(null, file); - }, 2000); - }); + return callback(null, file, 'gridfs'); } }); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js b/packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js index f97534577abf..0ba17089bfd0 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js @@ -1,47 +1,70 @@ -/* globals FileSystemStore:true, FileUpload, UploadFS, RocketChatFile */ +/* globals FileSystemStore:true, FileUpload, UploadFS, RocketChatFile, FileSystemStoreAvatar */ -const storeName = 'fileSystem'; +const transformWrite = function(readStream, writeStream, fileId, file) { + if (RocketChatFile.enabled === false || !/^image\/((x-windows-)?bmp|p?jpeg|png)$/.test(file.type)) { + return readStream.pipe(writeStream); + } + + let stream = undefined; + + const identify = function(err, data) { + if (err != null) { + return stream.pipe(writeStream); + } + + file.identify = { + format: data.format, + size: data.size + }; + + if ([null, undefined, '', 'Unknown', 'Undefined'].indexOf(data.Orientation) === -1) { + return RocketChatFile.gm(stream).autoOrient().stream().pipe(writeStream); + } else { + return stream.pipe(writeStream); + } + }; + + stream = RocketChatFile.gm(readStream).identify(identify).stream(); + return; +}; FileSystemStore = null; +FileSystemStoreAvatar = null; const createFileSystemStore = _.debounce(function() { const stores = UploadFS.getStores(); - if (stores[storeName]) { - delete stores[storeName]; - } + delete stores.fileSystem; + delete stores.fileSystemAvatar; + FileSystemStore = new UploadFS.store.Local({ - collection: RocketChat.models.Uploads.model, - name: storeName, path: RocketChat.settings.get('FileUpload_FileSystemPath'), //'/tmp/uploads/photos', + collection: RocketChat.models.Uploads.model, filter: new UploadFS.Filter({ onCheck: FileUpload.validateFileUpload }), - transformWrite(readStream, writeStream, fileId, file) { - if (RocketChatFile.enabled === false || !/^image\/((x-windows-)?bmp|p?jpeg|png)$/.test(file.type)) { - return readStream.pipe(writeStream); - } - - let stream = undefined; - - const identify = function(err, data) { - if (err != null) { - return stream.pipe(writeStream); - } - - file.identify = { - format: data.format, - size: data.size - }; + name: 'fileSystem', + transformWrite + }); - if ([null, undefined, '', 'Unknown', 'Undefined'].indexOf(data.Orientation) === -1) { - return RocketChatFile.gm(stream).autoOrient().stream().pipe(writeStream); - } else { - return stream.pipe(writeStream); + FileSystemStoreAvatar = new UploadFS.store.Local({ + path: RocketChat.settings.get('FileUpload_FileSystemPath'), //'/tmp/uploads/photos', + collection: RocketChat.models.Avatars.model, + name: 'fileSystemAvatar', + transformWrite: FileUpload.avatarTransformWrite, + onFinishUpload(file) { + // update file record to match user's username + const user = RocketChat.models.Users.findOneById(file.userId); + const oldAvatar = RocketChat.models.Avatars.findOneByName(user.username); + if (oldAvatar) { + try { + FileSystemStoreAvatar.delete(oldAvatar._id); + RocketChat.models.Avatars.deleteFile(oldAvatar._id); + } catch (e) { + console.error(e); } - }; - - stream = RocketChatFile.gm(readStream).identify(identify).stream(); - return; + } + RocketChat.models.Avatars.updateFileNameById(file._id, user.username); + // console.log('upload finished ->', file); } }); }, 500); @@ -50,7 +73,7 @@ RocketChat.settings.get('FileUpload_FileSystemPath', createFileSystemStore); const fs = Npm.require('fs'); -FileUpload.addHandler(storeName, { +FileUpload.addHandler('fileSystem', { get(file, req, res) { const filePath = FileSystemStore.getFilePath(file._id, file); @@ -77,3 +100,41 @@ FileUpload.addHandler(storeName, { return FileSystemStore.delete(file._id); } }); + +FileUpload.addHandler('fileSystemAvatar', { + get(file, req, res) { + const reqModifiedHeader = req.headers['if-modified-since']; + if (reqModifiedHeader) { + if (reqModifiedHeader === (file.uploadedAt && file.uploadedAt.toUTCString())) { + res.setHeader('Last-Modified', reqModifiedHeader); + res.writeHead(304); + res.end(); + return; + } + } + + const filePath = FileSystemStoreAvatar.getFilePath(file._id, file); + + try { + const stat = Meteor.wrapAsync(fs.stat)(filePath); + + if (stat && stat.isFile()) { + file = FileUpload.addExtensionTo(file); + res.setHeader('Content-Disposition', 'inline'); + res.setHeader('Last-Modified', file.uploadedAt.toUTCString()); + res.setHeader('Content-Type', file.type); + res.setHeader('Content-Length', file.size); + + FileSystemStoreAvatar.getReadStream(file._id, file).pipe(res); + } + } catch (e) { + res.writeHead(404); + res.end(); + return; + } + }, + + delete(file) { + return FileSystemStoreAvatar.delete(file._id); + } +}); diff --git a/packages/rocketchat-file-upload/server/lib/FileUpload.js b/packages/rocketchat-file-upload/server/lib/FileUpload.js index 05ec504af3cc..7e207eef10f7 100644 --- a/packages/rocketchat-file-upload/server/lib/FileUpload.js +++ b/packages/rocketchat-file-upload/server/lib/FileUpload.js @@ -41,3 +41,13 @@ FileUpload.addExtensionTo = function(file) { return file; }; + +FileUpload.avatarTransformWrite = function(readStream, writeStream, fileId, file) { + if (RocketChatFile.enabled === false || RocketChat.settings.get('Accounts_AvatarResize') !== true) { + return readStream.pipe(writeStream); + } + const height = RocketChat.settings.get('Accounts_AvatarSize'); + const width = height; + return RocketChatFile.gm(readStream).background('#ffffff').resize(width, `${ height }^`).gravity('Center').crop(width, height).extent(width, height).stream('jpeg').pipe(writeStream); +}; + diff --git a/packages/rocketchat-google-natural-language/.npm/package/npm-shrinkwrap.json b/packages/rocketchat-google-natural-language/.npm/package/npm-shrinkwrap.json index a34613259049..c931564c628f 100644 --- a/packages/rocketchat-google-natural-language/.npm/package/npm-shrinkwrap.json +++ b/packages/rocketchat-google-natural-language/.npm/package/npm-shrinkwrap.json @@ -1,8 +1,8 @@ { "dependencies": { "ajv": { - "version": "4.11.7", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-4.11.7.tgz", + "version": "4.11.8", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-4.11.8.tgz", "from": "ajv@>=4.9.1 <5.0.0" }, "ansi-regex": { @@ -30,11 +30,6 @@ "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", "from": "arrify@>=1.0.1 <2.0.0" }, - "ascli": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/ascli/-/ascli-1.0.1.tgz", - "from": "ascli@>=1.0.0 <2.0.0" - }, "asn1": { "version": "0.2.3", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz", @@ -46,8 +41,8 @@ "from": "assert-plus@>=0.2.0 <0.3.0" }, "async": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/async/-/async-2.3.0.tgz", + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/async/-/async-2.4.0.tgz", "from": "async@>=2.1.2 <3.0.0" }, "asynckit": { @@ -65,11 +60,6 @@ "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.6.0.tgz", "from": "aws4@>=1.2.1 <2.0.0" }, - "balanced-match": { - "version": "0.4.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-0.4.2.tgz", - "from": "balanced-match@>=0.4.1 <0.5.0" - }, "base64url": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/base64url/-/base64url-2.0.0.tgz", @@ -85,11 +75,6 @@ "resolved": "https://registry.npmjs.org/boom/-/boom-2.10.1.tgz", "from": "boom@>=2.0.0 <3.0.0" }, - "brace-expansion": { - "version": "1.1.7", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.7.tgz", - "from": "brace-expansion@>=1.0.0 <2.0.0" - }, "buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", @@ -100,16 +85,6 @@ "resolved": "https://registry.npmjs.org/buffer-shims/-/buffer-shims-1.0.0.tgz", "from": "buffer-shims@>=1.0.0 <1.1.0" }, - "bytebuffer": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/bytebuffer/-/bytebuffer-5.0.1.tgz", - "from": "bytebuffer@>=5.0.0 <6.0.0" - }, - "camelcase": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-2.1.1.tgz", - "from": "camelcase@>=2.0.1 <3.0.0" - }, "capture-stack-trace": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/capture-stack-trace/-/capture-stack-trace-1.0.0.tgz", @@ -125,26 +100,11 @@ "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", "from": "chalk@>=1.1.1 <2.0.0" }, - "cliui": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-3.2.0.tgz", - "from": "cliui@>=3.0.3 <4.0.0" - }, "co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", "from": "co@>=4.6.0 <5.0.0" }, - "code-point-at": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", - "from": "code-point-at@>=1.0.0 <2.0.0" - }, - "colour": { - "version": "0.7.1", - "resolved": "https://registry.npmjs.org/colour/-/colour-0.7.1.tgz", - "from": "colour@>=0.7.1 <0.8.0" - }, "combined-stream": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.5.tgz", @@ -155,11 +115,6 @@ "resolved": "https://registry.npmjs.org/commander/-/commander-2.9.0.tgz", "from": "commander@>=2.9.0 <3.0.0" }, - "concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "from": "concat-map@0.0.1" - }, "concat-stream": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.0.tgz", @@ -192,11 +147,6 @@ } } }, - "decamelize": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", - "from": "decamelize@>=1.1.1 <2.0.0" - }, "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -233,8 +183,8 @@ "from": "escape-string-regexp@>=1.0.2 <2.0.0" }, "extend": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.0.tgz", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.1.tgz", "from": "extend@>=3.0.0 <4.0.0" }, "extsprintf": { @@ -252,11 +202,6 @@ "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.1.4.tgz", "from": "form-data@>=2.1.1 <2.2.0" }, - "fs.realpath": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "from": "fs.realpath@>=1.0.0 <2.0.0" - }, "generate-function": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.0.0.tgz", @@ -279,11 +224,6 @@ } } }, - "glob": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.1.tgz", - "from": "glob@>=7.0.5 <8.0.0" - }, "google-auth-library": { "version": "0.10.0", "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-0.10.0.tgz", @@ -322,8 +262,8 @@ "from": "graceful-readlink@>=1.0.0" }, "grpc": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/grpc/-/grpc-1.2.4.tgz", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/grpc/-/grpc-1.3.0.tgz", "from": "grpc@>=1.1.0 <2.0.0", "dependencies": { "node-pre-gyp": { @@ -378,8 +318,8 @@ "from": "npmlog@>=4.0.2 <5.0.0", "dependencies": { "are-we-there-yet": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.2.tgz", + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.4.tgz", "from": "are-we-there-yet@>=1.1.2 <1.2.0", "dependencies": { "delegates": { @@ -390,7 +330,7 @@ "readable-stream": { "version": "2.2.9", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.2.9.tgz", - "from": "readable-stream@>=2.0.0 <3.0.0||>=1.1.13 <2.0.0", + "from": "readable-stream@>=2.0.6 <3.0.0", "dependencies": { "buffer-shims": { "version": "1.0.0", @@ -437,8 +377,8 @@ "from": "console-control-strings@>=1.1.0 <1.2.0" }, "gauge": { - "version": "2.7.3", - "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.3.tgz", + "version": "2.7.4", + "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", "from": "gauge@>=2.7.1 <2.8.0", "dependencies": { "aproba": { @@ -598,8 +538,8 @@ "from": "har-validator@>=4.2.1 <4.3.0", "dependencies": { "ajv": { - "version": "4.11.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-4.11.6.tgz", + "version": "4.11.7", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-4.11.7.tgz", "from": "ajv@>=4.9.1 <5.0.0", "dependencies": { "co": { @@ -693,8 +633,8 @@ } }, "sshpk": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.11.0.tgz", + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.13.0.tgz", "from": "sshpk@>=1.7.0 <2.0.0", "dependencies": { "asn1": { @@ -723,8 +663,8 @@ "from": "ecc-jsbn@>=0.1.1 <0.2.0" }, "getpass": { - "version": "0.1.6", - "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.6.tgz", + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", "from": "getpass@>=0.1.1 <0.2.0" }, "jodid25519": { @@ -939,14 +879,14 @@ "from": "tar-pack@>=3.4.0 <4.0.0", "dependencies": { "debug": { - "version": "2.6.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.3.tgz", + "version": "2.6.5", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.5.tgz", "from": "debug@>=2.2.0 <3.0.0", "dependencies": { "ms": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.2.tgz", - "from": "ms@0.7.2" + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.3.tgz", + "from": "ms@0.7.3" } } }, @@ -1006,7 +946,7 @@ "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "from": "once@>=1.0.0 <2.0.0", + "from": "once@>=1.3.3 <2.0.0", "dependencies": { "wrappy": { "version": "1.0.2", @@ -1103,31 +1043,16 @@ "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.1.1.tgz", "from": "http-signature@>=1.1.0 <1.2.0" }, - "inflight": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "from": "inflight@>=1.0.4 <2.0.0" - }, "inherits": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", "from": "inherits@>=2.0.3 <3.0.0" }, - "invert-kv": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz", - "from": "invert-kv@>=1.0.0 <2.0.0" - }, "is": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/is/-/is-3.2.1.tgz", "from": "is@>=3.0.1 <4.0.0" }, - "is-fullwidth-code-point": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", - "from": "is-fullwidth-code-point@>=1.0.0 <2.0.0" - }, "is-my-json-valid": { "version": "2.16.0", "resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.16.0.tgz", @@ -1215,11 +1140,6 @@ "resolved": "https://registry.npmjs.org/jws/-/jws-3.1.4.tgz", "from": "jws@>=3.1.4 <4.0.0" }, - "lcid": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz", - "from": "lcid@>=1.0.0 <2.0.0" - }, "lodash": { "version": "4.17.4", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.4.tgz", @@ -1238,7 +1158,7 @@ "long": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/long/-/long-3.2.0.tgz", - "from": "long@>=3.0.0 <4.0.0" + "from": "long@>=3.2.0 <4.0.0" }, "methmeth": { "version": "1.1.0", @@ -1260,11 +1180,6 @@ "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.15.tgz", "from": "mime-types@>=2.1.7 <2.2.0" }, - "minimatch": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.3.tgz", - "from": "minimatch@>=3.0.2 <4.0.0" - }, "modelo": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/modelo/-/modelo-4.2.0.tgz", @@ -1280,11 +1195,6 @@ "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.7.1.tgz", "from": "node-forge@>=0.7.1 <0.8.0" }, - "number-is-nan": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", - "from": "number-is-nan@>=1.0.0 <2.0.0" - }, "oauth-sign": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz", @@ -1300,21 +1210,6 @@ "resolved": "https://registry.npmjs.org/once/-/once-1.3.3.tgz", "from": "once@>=1.3.0 <1.4.0" }, - "optjs": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/optjs/-/optjs-3.2.2.tgz", - "from": "optjs@>=3.2.2 <3.3.0" - }, - "os-locale": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz", - "from": "os-locale@>=1.4.0 <2.0.0" - }, - "path-is-absolute": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "from": "path-is-absolute@>=1.0.0 <2.0.0" - }, "performance-now": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-0.2.0.tgz", @@ -1341,9 +1236,9 @@ "from": "propprop@>=0.3.1 <0.4.0" }, "protobufjs": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-5.0.2.tgz", - "from": "protobufjs@>=5.0.0 <6.0.0" + "version": "6.7.3", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.7.3.tgz", + "from": "protobufjs@>=6.7.0 <7.0.0" }, "punycode": { "version": "1.4.1", @@ -1451,11 +1346,6 @@ "resolved": "https://registry.npmjs.org/string-format-obj/-/string-format-obj-1.1.0.tgz", "from": "string-format-obj@>=1.1.0 <2.0.0" }, - "string-width": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", - "from": "string-width@>=1.0.1 <2.0.0" - }, "string_decoder": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.0.tgz", @@ -1521,16 +1411,6 @@ "resolved": "https://registry.npmjs.org/verror/-/verror-1.3.6.tgz", "from": "verror@1.3.6" }, - "window-size": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/window-size/-/window-size-0.1.4.tgz", - "from": "window-size@>=0.1.4 <0.2.0" - }, - "wrap-ansi": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz", - "from": "wrap-ansi@>=2.0.0 <3.0.0" - }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -1540,16 +1420,6 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", "from": "xtend@>=4.0.0 <5.0.0" - }, - "y18n": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.1.tgz", - "from": "y18n@>=3.2.0 <4.0.0" - }, - "yargs": { - "version": "3.32.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-3.32.0.tgz", - "from": "yargs@>=3.10.0 <4.0.0" } } } diff --git a/packages/rocketchat-ui/client/lib/fileUpload.js b/packages/rocketchat-ui/client/lib/fileUpload.js index d7ef1590a437..20d2007951c0 100644 --- a/packages/rocketchat-ui/client/lib/fileUpload.js +++ b/packages/rocketchat-ui/client/lib/fileUpload.js @@ -145,7 +145,7 @@ fileUpload = function(filesToUpload) { description: document.getElementById('file-description').value }; - const upload = fileUploadHandler('rocketchat-uploads-gs', record, file.file); + const upload = fileUploadHandler('upload', record, file.file); let uploading = Session.get('uploading') || []; uploading.push({ @@ -166,7 +166,7 @@ fileUpload = function(filesToUpload) { } }; - upload.start(function(error, file) { + upload.start(function(error, file, storage) { if (error) { let uploading = Session.get('uploading'); if (!Array.isArray(uploading)) { @@ -187,14 +187,21 @@ fileUpload = function(filesToUpload) { Session.set('uploading', uploading); return; - } else if (file) { - const uploading = Session.get('uploading'); - if (uploading !== null) { - const item = _.findWhere(uploading, { - id: this.id - }); - return Session.set('uploading', _.without(uploading, item)); - } + } + + + if (file) { + Meteor.call('sendFileMessage', roomId, storage, file, () => { + Meteor.setTimeout(() => { + const uploading = Session.get('uploading'); + if (uploading !== null) { + const item = _.findWhere(uploading, { + id: this.id + }); + return Session.set('uploading', _.without(uploading, item)); + } + }, 2000); + }); } }); From 536fba74d7129cc4149f56cfb4408622b858f369 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Mon, 1 May 2017 18:31:09 -0300 Subject: [PATCH 08/42] Avatar: Refactor some code, allow set avatar from server for FS and GridFS --- lib/fileUpload.js | 4 +- .../client/lib/FileUploadFileSystem.js | 4 +- .../server/config/configFileUploadAmazonS3.js | 8 +- .../config/configFileUploadFileSystem.js | 168 +++++++++--------- .../config/configFileUploadGoogleStorage.js | 12 +- .../server/config/configFileUploadGridFS.js | 39 ++-- .../server/lib/FileUpload.js | 126 +++++++++---- .../server/functions/deleteMessage.js | 2 +- .../server/functions/deleteUser.js | 3 +- .../server/functions/setUserAvatar.js | 15 +- server/methods/deleteFileMessage.js | 2 +- server/methods/resetAvatar.js | 2 +- server/startup/initialData.js | 19 +- 13 files changed, 250 insertions(+), 154 deletions(-) diff --git a/lib/fileUpload.js b/lib/fileUpload.js index 782015a73cd5..f5eafb6e0985 100644 --- a/lib/fileUpload.js +++ b/lib/fileUpload.js @@ -11,7 +11,7 @@ if (UploadFS) { Meteor.fileStore = new UploadFS.store.GridFS({ collection: RocketChat.models.Uploads.model, - name: 'rocketchat_uploads', + name: 'GridFS:Uploads', collectionName: 'rocketchat_uploads', filter: new UploadFS.Filter({ onCheck: FileUpload.validateFileUpload @@ -75,7 +75,7 @@ if (UploadFS) { Meteor.fileStoreAvatar = new UploadFS.store.GridFS({ collection: RocketChat.models.Avatars.model, - name: 'rocketchat_uploads_avatar', + name: 'GridFS:Avatars', collectionName: 'rocketchat_avatars', // filter: new UploadFS.Filter({ // onCheck: FileUpload.validateFileUpload diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadFileSystem.js b/packages/rocketchat-file-upload/client/lib/FileUploadFileSystem.js index 025a3cf78103..10bb44cf74fb 100644 --- a/packages/rocketchat-file-upload/client/lib/FileUploadFileSystem.js +++ b/packages/rocketchat-file-upload/client/lib/FileUploadFileSystem.js @@ -2,7 +2,7 @@ FileSystemStore = new UploadFS.store.Local({ collection: RocketChat.models.Uploads.model, - name: 'fileSystem', + name: 'FileSystem:Uploads', filter: new UploadFS.Filter({ onCheck: FileUpload.validateFileUpload }) @@ -10,7 +10,7 @@ FileSystemStore = new UploadFS.store.Local({ FileSystemStoreAvatar = new UploadFS.store.Local({ collection: RocketChat.models.Avatars.model, - name: 'fileSystemAvatar', + name: 'FileSystem:Avatars', filter: new UploadFS.Filter({ onCheck: FileUpload.validateFileUpload }) diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js index a1844ba22e06..fe18ecaacebe 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js @@ -1,5 +1,6 @@ /* globals Slingshot, FileUpload, AWS */ -const crypto = Npm.require('crypto'); +import crypto from 'crypto'; +import { FileUploadClass } from '../lib/FileUpload'; let S3accessKey; let S3secretKey; @@ -16,7 +17,9 @@ const generateURL = function(file) { return `${ file.url }?AWSAccessKeyId=${ encodeURIComponent(S3accessKey) }&Expires=${ expires }&Signature=${ encodeURIComponent(signature) }`; }; -FileUpload.addHandler('s3', { +new FileUploadClass({ + name: 'S3:Uploads', + get(file, req, res) { const fileUrl = generateURL(file); @@ -26,6 +29,7 @@ FileUpload.addHandler('s3', { } res.end(); }, + delete(file) { const s3 = new AWS.S3(); const request = s3.deleteObject({ diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js b/packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js index 0ba17089bfd0..9ffdb26a7a9c 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js @@ -1,81 +1,20 @@ -/* globals FileSystemStore:true, FileUpload, UploadFS, RocketChatFile, FileSystemStoreAvatar */ +/* globals FileUpload, UploadFS, RocketChatFile */ -const transformWrite = function(readStream, writeStream, fileId, file) { - if (RocketChatFile.enabled === false || !/^image\/((x-windows-)?bmp|p?jpeg|png)$/.test(file.type)) { - return readStream.pipe(writeStream); - } +import fs from 'fs'; +import { FileUploadClass } from '../lib/FileUpload'; - let stream = undefined; - - const identify = function(err, data) { - if (err != null) { - return stream.pipe(writeStream); - } - - file.identify = { - format: data.format, - size: data.size - }; +const insert = function(file, stream, cb) { + const fileId = this.store.create(file); - if ([null, undefined, '', 'Unknown', 'Undefined'].indexOf(data.Orientation) === -1) { - return RocketChatFile.gm(stream).autoOrient().stream().pipe(writeStream); - } else { - return stream.pipe(writeStream); - } - }; - - stream = RocketChatFile.gm(readStream).identify(identify).stream(); - return; + this.store.write(stream, fileId, cb); }; -FileSystemStore = null; -FileSystemStoreAvatar = null; +const FileSystemUploads = new FileUploadClass({ + name: 'FileSystem:Uploads', + // store setted bellow -const createFileSystemStore = _.debounce(function() { - const stores = UploadFS.getStores(); - delete stores.fileSystem; - delete stores.fileSystemAvatar; - - FileSystemStore = new UploadFS.store.Local({ - path: RocketChat.settings.get('FileUpload_FileSystemPath'), //'/tmp/uploads/photos', - collection: RocketChat.models.Uploads.model, - filter: new UploadFS.Filter({ - onCheck: FileUpload.validateFileUpload - }), - name: 'fileSystem', - transformWrite - }); - - FileSystemStoreAvatar = new UploadFS.store.Local({ - path: RocketChat.settings.get('FileUpload_FileSystemPath'), //'/tmp/uploads/photos', - collection: RocketChat.models.Avatars.model, - name: 'fileSystemAvatar', - transformWrite: FileUpload.avatarTransformWrite, - onFinishUpload(file) { - // update file record to match user's username - const user = RocketChat.models.Users.findOneById(file.userId); - const oldAvatar = RocketChat.models.Avatars.findOneByName(user.username); - if (oldAvatar) { - try { - FileSystemStoreAvatar.delete(oldAvatar._id); - RocketChat.models.Avatars.deleteFile(oldAvatar._id); - } catch (e) { - console.error(e); - } - } - RocketChat.models.Avatars.updateFileNameById(file._id, user.username); - // console.log('upload finished ->', file); - } - }); -}, 500); - -RocketChat.settings.get('FileUpload_FileSystemPath', createFileSystemStore); - -const fs = Npm.require('fs'); - -FileUpload.addHandler('fileSystem', { get(file, req, res) { - const filePath = FileSystemStore.getFilePath(file._id, file); + const filePath = this.store.getFilePath(file._id, file); try { const stat = Meteor.wrapAsync(fs.stat)(filePath); @@ -87,7 +26,7 @@ FileUpload.addHandler('fileSystem', { res.setHeader('Content-Type', file.type); res.setHeader('Content-Length', file.size); - FileSystemStore.getReadStream(file._id, file).pipe(res); + this.store.getReadStream(file._id, file).pipe(res); } } catch (e) { res.writeHead(404); @@ -96,12 +35,13 @@ FileUpload.addHandler('fileSystem', { } }, - delete(file) { - return FileSystemStore.delete(file._id); - } + insert }); -FileUpload.addHandler('fileSystemAvatar', { +const FileSystemAvatars = new FileUploadClass({ + name: 'FileSystem:Avatars', + // store setted bellow + get(file, req, res) { const reqModifiedHeader = req.headers['if-modified-since']; if (reqModifiedHeader) { @@ -113,7 +53,7 @@ FileUpload.addHandler('fileSystemAvatar', { } } - const filePath = FileSystemStoreAvatar.getFilePath(file._id, file); + const filePath = this.store.getFilePath(file._id, file); try { const stat = Meteor.wrapAsync(fs.stat)(filePath); @@ -125,7 +65,7 @@ FileUpload.addHandler('fileSystemAvatar', { res.setHeader('Content-Type', file.type); res.setHeader('Content-Length', file.size); - FileSystemStoreAvatar.getReadStream(file._id, file).pipe(res); + this.store.getReadStream(file._id, file).pipe(res); } } catch (e) { res.writeHead(404); @@ -134,7 +74,73 @@ FileUpload.addHandler('fileSystemAvatar', { } }, - delete(file) { - return FileSystemStoreAvatar.delete(file._id); - } + insert }); + + +const transformWrite = function(readStream, writeStream, fileId, file) { + if (RocketChatFile.enabled === false || !/^image\/((x-windows-)?bmp|p?jpeg|png)$/.test(file.type)) { + return readStream.pipe(writeStream); + } + + let stream = undefined; + + const identify = function(err, data) { + if (err != null) { + return stream.pipe(writeStream); + } + + file.identify = { + format: data.format, + size: data.size + }; + + if ([null, undefined, '', 'Unknown', 'Undefined'].indexOf(data.Orientation) === -1) { + return RocketChatFile.gm(stream).autoOrient().stream().pipe(writeStream); + } else { + return stream.pipe(writeStream); + } + }; + + stream = RocketChatFile.gm(readStream).identify(identify).stream(); + return; +}; + +const createFileSystemStore = _.debounce(function() { + const stores = UploadFS.getStores(); + delete stores['FileSystem:Uploads']; + delete stores['FileSystem:Avatars']; + + FileSystemUploads.store = new UploadFS.store.Local({ + path: RocketChat.settings.get('FileUpload_FileSystemPath'), //'/tmp/uploads/photos', + collection: FileSystemUploads.model.model, + filter: new UploadFS.Filter({ + onCheck: FileUpload.validateFileUpload + }), + name: FileSystemUploads.name, + transformWrite + }); + + FileSystemAvatars.store = new UploadFS.store.Local({ + path: RocketChat.settings.get('FileUpload_FileSystemPath'), //'/tmp/uploads/photos', + collection: FileSystemAvatars.model.model, + name: FileSystemAvatars.name, + transformWrite: FileUpload.avatarTransformWrite, + onFinishUpload(file) { + // update file record to match user's username + const user = RocketChat.models.Users.findOneById(file.userId); + const oldAvatar = FileSystemAvatars.model.findOneByName(user.username); + if (oldAvatar) { + try { + FileSystemAvatars.deleteById(oldAvatar._id); + } catch (e) { + console.error(e); + } + } + FileSystemAvatars.model.updateFileNameById(file._id, user.username); + // console.log('upload finished ->', file); + } + }); +}, 500); + +RocketChat.settings.get('FileUpload_FileSystemPath', createFileSystemStore); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js index e7a2b7b4fc8e..69e0b514ba0c 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js @@ -1,6 +1,7 @@ -/* globals FileUpload, Slingshot, SystemLogger */ +/* globals FileUpload, Slingshot */ -const crypto = Npm.require('crypto'); +import crypto from 'crypto'; +import { FileUploadClass } from '../lib/FileUpload'; function generateUrlParts({ file }) { const accessId = RocketChat.settings.get('FileUpload_GoogleStorage_AccessId'); @@ -62,7 +63,9 @@ function createDirective(directiveName, { key, bucket, accessId, secret }) { } } -FileUpload.addHandler('googleCloudStorage', { +new FileUploadClass({ + name: 'googleCloudStorage', + get(file, req, res) { const fileUrl = generateGetURL({ file }); @@ -72,12 +75,15 @@ FileUpload.addHandler('googleCloudStorage', { } res.end(); }, + delete(file) { if (!file || !file.googleCloudStorage) { console.warn('Failed to delete a file which is uploaded to Google Cloud Storage, the file and googleCloudStorage properties are not defined.'); return; } + // RocketChat.models.Uploads.deleteFile(file._id); + const url = generateDeleteUrl({ file }); if (_.isEmpty(url)) { diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js b/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js index fad40155182d..4e7553f642e2 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js @@ -1,7 +1,10 @@ /* globals FileUpload, UploadFS */ -const stream = Npm.require('stream'); -const zlib = Npm.require('zlib'); -const util = Npm.require('util'); +import stream from 'stream'; +import zlib from 'zlib'; +import util from 'util'; + +import { FileUploadClass } from '../lib/FileUpload'; + const logger = new Logger('FileUpload'); function ExtractRange(options) { @@ -124,7 +127,18 @@ const readFromGridFS = function(storeName, fileId, file, headers, req, res) { } }; -FileUpload.addHandler('rocketchat_uploads', { +const insert = function(file, stream, cb) { + const fileId = this.store.create(file); + + this.store.write(stream, fileId, cb); +}; + +new FileUploadClass({ + name: 'GridFS:Uploads', + getStore() { + return Meteor.fileStore; + }, + get(file, req, res) { file = FileUpload.addExtensionTo(file); const headers = { @@ -135,12 +149,16 @@ FileUpload.addHandler('rocketchat_uploads', { }; return readFromGridFS(file.store, file._id, file, headers, req, res); }, - delete(file) { - return Meteor.fileStore.delete(file._id); - } + + insert }); -FileUpload.addHandler('rocketchat_uploads_avatar', { +new FileUploadClass({ + name: 'GridFS:Avatars', + getStore() { + return Meteor.fileStoreAvatar + }, + get(file, req, res) { const reqModifiedHeader = req.headers['if-modified-since']; if (reqModifiedHeader) { @@ -163,7 +181,6 @@ FileUpload.addHandler('rocketchat_uploads_avatar', { }; return readFromGridFS(file.store, file._id, file, headers, req, res); }, - delete(file) { - return Meteor.fileStore.delete(file._id); - } + + insert }); diff --git a/packages/rocketchat-file-upload/server/lib/FileUpload.js b/packages/rocketchat-file-upload/server/lib/FileUpload.js index 7e207eef10f7..f606d0ad0d4e 100644 --- a/packages/rocketchat-file-upload/server/lib/FileUpload.js +++ b/packages/rocketchat-file-upload/server/lib/FileUpload.js @@ -1,53 +1,109 @@ -/* globals FileUpload:true */ import mime from 'mime-type/with-db'; -FileUpload.handlers = {}; +Object.assign(FileUpload, { + handlers: {}, -FileUpload.addHandler = function(store, handler) { - this.handlers[store] = handler; -}; + avatarTransformWrite(readStream, writeStream/*, fileId, file*/) { + if (RocketChatFile.enabled === false || RocketChat.settings.get('Accounts_AvatarResize') !== true) { + return readStream.pipe(writeStream); + } + const height = RocketChat.settings.get('Accounts_AvatarSize'); + const width = height; + return RocketChatFile.gm(readStream).background('#ffffff').resize(width, `${ height }^`).gravity('Center').crop(width, height).extent(width, height).stream('jpeg').pipe(writeStream); + }, -FileUpload.delete = function(fileId) { - const file = RocketChat.models.Uploads.findOneById(fileId); + addExtensionTo(file) { + if (mime.lookup(file.name) === file.type) { + return file; + } - if (!file) { - return; + const ext = mime.extension(file.type); + if (ext && false === new RegExp(`\.${ ext }$`, 'i').test(file.name)) { + file.name = `${ file.name }.${ ext }`; + } + + return file; + }, + + getStore(modelName) { + const storageType = RocketChat.settings.get('FileUpload_Storage_Type'); + const handlerName = `${ storageType }:${ modelName }`; + + if (this.handlers[handlerName] == null) { + console.error(`Upload handler "${ handlerName }" does not exists`); + } + + return this.handlers[handlerName]; + }, + + get(file, req, res, next) { + if (file.store && this.handlers && this.handlers[file.store] && this.handlers[file.store].get) { + this.handlers[file.store].get(file, req, res, next); + } else { + res.writeHead(404); + res.end(); + return; + } } +}); + - this.handlers[file.store].delete(file); +export class FileUploadClass { + constructor({ name, model, store, get, insert, getStore }) { + this.name = name; + this.model = model || this.getModelFromName(); + this._store = store; + this.get = get; + this.insert = insert; - return RocketChat.models.Uploads.deleteFile(file._id); -}; + if (getStore) { + this.getStore = getStore; + } -FileUpload.get = function(file, req, res, next) { - if (file.store && this.handlers && this.handlers[file.store] && this.handlers[file.store].get) { - this.handlers[file.store].get.call(this, file, req, res, next); - } else { - res.writeHead(404); - res.end(); - return; + FileUpload.handlers[name] = this; } -}; -FileUpload.addExtensionTo = function(file) { - if (mime.lookup(file.name) === file.type) { - return file; + getStore() { + return this._store; } - const ext = mime.extension(file.type); - if (ext && false === new RegExp(`\.${ ext }$`, 'i').test(file.name)) { - file.name = `${ file.name }.${ ext }`; + get store() { + return this.getStore(); } - return file; -}; + set store(store) { + this._store = store; + } -FileUpload.avatarTransformWrite = function(readStream, writeStream, fileId, file) { - if (RocketChatFile.enabled === false || RocketChat.settings.get('Accounts_AvatarResize') !== true) { - return readStream.pipe(writeStream); + getModelFromName() { + return RocketChat.models[this.name.split(':')[1]]; } - const height = RocketChat.settings.get('Accounts_AvatarSize'); - const width = height; - return RocketChatFile.gm(readStream).background('#ffffff').resize(width, `${ height }^`).gravity('Center').crop(width, height).extent(width, height).stream('jpeg').pipe(writeStream); -}; + delete(fileId) { + if (this.store && this.store.delete) { + this.store.delete(fileId); + } + + return this.model.deleteFile(fileId); + } + + deleteById(fileId) { + const file = this.model.findOneById(fileId); + + if (!file) { + return; + } + + return this.delete(file._id); + } + + deleteByName(fileName) { + const file = this.model.findOneByName(fileName); + + if (!file) { + return; + } + + return this.delete(file._id); + } +} diff --git a/packages/rocketchat-lib/server/functions/deleteMessage.js b/packages/rocketchat-lib/server/functions/deleteMessage.js index 7a190964be11..50fd5999864e 100644 --- a/packages/rocketchat-lib/server/functions/deleteMessage.js +++ b/packages/rocketchat-lib/server/functions/deleteMessage.js @@ -21,7 +21,7 @@ RocketChat.deleteMessage = function(message, user) { } if (message.file && message.file._id) { - FileUpload.delete(message.file._id); + FileUpload.getStore('Uploads').deleteById(message.file._id); } Meteor.defer(function() { diff --git a/packages/rocketchat-lib/server/functions/deleteUser.js b/packages/rocketchat-lib/server/functions/deleteUser.js index a139389b064d..ab7ec6719b94 100644 --- a/packages/rocketchat-lib/server/functions/deleteUser.js +++ b/packages/rocketchat-lib/server/functions/deleteUser.js @@ -1,4 +1,3 @@ -/* globals RocketChat */ RocketChat.deleteUser = function(userId) { const user = RocketChat.models.Users.findOneById(userId); @@ -22,7 +21,7 @@ RocketChat.deleteUser = function(userId) { // removes user's avatar if (user.avatarOrigin === 'upload' || user.avatarOrigin === 'url') { - RocketChatFileAvatarInstance.deleteFile(encodeURIComponent(`${ user.username }.jpg`)); + FileUpload.getStore('Avatars').deleteByName(user.username); } RocketChat.models.Integrations.disableByUserId(userId); // Disables all the integrations which rely on the user being deleted. diff --git a/packages/rocketchat-lib/server/functions/setUserAvatar.js b/packages/rocketchat-lib/server/functions/setUserAvatar.js index e80a8c7cdd70..40289080a947 100644 --- a/packages/rocketchat-lib/server/functions/setUserAvatar.js +++ b/packages/rocketchat-lib/server/functions/setUserAvatar.js @@ -40,13 +40,18 @@ RocketChat.setUserAvatar = function(user, dataURI, contentType, service) { } const rs = RocketChatFile.bufferToStream(new Buffer(image, encoding)); - RocketChatFileAvatarInstance.deleteFile(encodeURIComponent(`${ user.username }.jpg`)); - const ws = RocketChatFileAvatarInstance.createWriteStream(encodeURIComponent(`${ user.username }.jpg`), contentType); - ws.on('end', Meteor.bindEnvironment(function() { + const fileStore = FileUpload.getStore('Avatars'); + fileStore.deleteByName(user.username); + + const file = { + userId: user._id, + type: contentType + }; + + fileStore.insert(file, rs, () => { Meteor.setTimeout(function() { RocketChat.models.Users.setAvatarOrigin(user._id, service); RocketChat.Notifications.notifyLogged('updateAvatar', {username: user.username}); }, 500); - })); - rs.pipe(ws); + }); }; diff --git a/server/methods/deleteFileMessage.js b/server/methods/deleteFileMessage.js index ce10e596c459..9ffc1fe7f8a6 100644 --- a/server/methods/deleteFileMessage.js +++ b/server/methods/deleteFileMessage.js @@ -9,6 +9,6 @@ Meteor.methods({ return Meteor.call('deleteMessage', msg); } - return FileUpload.delete(fileID); + return FileUpload.getStore('Uploads').deleteById(fileID); } }); diff --git a/server/methods/resetAvatar.js b/server/methods/resetAvatar.js index 8b5d1fed2191..ea551e50850f 100644 --- a/server/methods/resetAvatar.js +++ b/server/methods/resetAvatar.js @@ -13,7 +13,7 @@ Meteor.methods({ } const user = Meteor.user(); - RocketChatFileAvatarInstance.deleteFile(`${ user.username }.jpg`); + FileUpload.getStore('Avatars').deleteByName(user.username); RocketChat.models.Users.unsetAvatarOrigin(user._id); RocketChat.Notifications.notifyLogged('updateAvatar', { username: user.username diff --git a/server/startup/initialData.js b/server/startup/initialData.js index 298d76c19119..b8f9acdcaf37 100644 --- a/server/startup/initialData.js +++ b/server/startup/initialData.js @@ -21,16 +21,19 @@ Meteor.startup(function() { RocketChat.authz.addUserRoles('rocket.cat', 'bot'); const rs = RocketChatFile.bufferToStream(new Buffer(Assets.getBinary('avatars/rocketcat.png'), 'utf8')); + const fileStore = FileUpload.getStore('Avatars'); + fileStore.deleteByName('rocket.cat'); - RocketChatFileAvatarInstance.deleteFile('rocket.cat.jpg'); - - const ws = RocketChatFileAvatarInstance.createWriteStream('rocket.cat.jpg', 'image/png'); - - ws.on('end', Meteor.bindEnvironment(function() { - return RocketChat.models.Users.setAvatarOrigin('rocket.cat', 'local'); - })); + const file = { + userId: 'rocket.cat', + type: 'image/png' + }; - rs.pipe(ws); + Meteor.runAsUser('rocket.cat', () => { + fileStore.insert(file, rs, () => { + return RocketChat.models.Users.setAvatarOrigin('rocket.cat', 'local'); + }); + }); } if (process.env.ADMIN_PASS) { From 001dd3a72721c2fa654e5c50cd64de6d3a35691d Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Tue, 2 May 2017 09:19:21 -0300 Subject: [PATCH 09/42] Add FileUpload to .eslintrc --- .eslintrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.eslintrc b/.eslintrc index bbae28a07d60..2c1846603505 100644 --- a/.eslintrc +++ b/.eslintrc @@ -105,6 +105,7 @@ "EJSON" : false, "Email" : false, "FlowRouter" : false, + "FileUpload" : false, "HTTP" : false, "getNextAgent" : false, "handleError" : false, From 8360b869b875ff0b31ede1ee602f40ed86a028b0 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Tue, 2 May 2017 09:20:14 -0300 Subject: [PATCH 10/42] Fix ESLint error --- .../server/config/configFileUploadGridFS.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js b/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js index 4e7553f642e2..e26b7cc7d6b4 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js @@ -156,7 +156,7 @@ new FileUploadClass({ new FileUploadClass({ name: 'GridFS:Avatars', getStore() { - return Meteor.fileStoreAvatar + return Meteor.fileStoreAvatar; }, get(file, req, res) { From 06d966cd8bf049b9c6abbbd088dc93b09e341fe0 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Tue, 2 May 2017 10:09:02 -0300 Subject: [PATCH 11/42] Remove RocketChatFileAvatarInstance --- packages/rocketchat-ldap/server/sync.js | 16 ++++++--- .../server/functions/setUsername.coffee | 10 ++---- server/startup/avatar.js | 34 ------------------- server/startup/migrations/v002.js | 14 +++++--- 4 files changed, 23 insertions(+), 51 deletions(-) diff --git a/packages/rocketchat-ldap/server/sync.js b/packages/rocketchat-ldap/server/sync.js index df3ec4bf828b..cdac0c769176 100644 --- a/packages/rocketchat-ldap/server/sync.js +++ b/packages/rocketchat-ldap/server/sync.js @@ -143,16 +143,22 @@ syncUserData = function syncUserData(user, ldapUser) { const avatar = ldapUser.raw.thumbnailPhoto || ldapUser.raw.jpegPhoto; if (avatar) { logger.info('Syncing user avatar'); + const rs = RocketChatFile.bufferToStream(avatar); - RocketChatFileAvatarInstance.deleteFile(encodeURIComponent(`${ user.username }.jpg`)); - const ws = RocketChatFileAvatarInstance.createWriteStream(encodeURIComponent(`${ user.username }.jpg`), 'image/jpeg'); - ws.on('end', Meteor.bindEnvironment(function() { + const fileStore = FileUpload.getStore('Avatars'); + fileStore.deleteByName(user.username); + + const file = { + userId: user._id, + type: 'image/jpeg' + }; + + fileStore.insert(file, rs, () => { Meteor.setTimeout(function() { RocketChat.models.Users.setAvatarOrigin(user._id, 'ldap'); RocketChat.Notifications.notifyLogged('updateAvatar', {username: user.username}); }, 500); - })); - rs.pipe(ws); + }); } } }; diff --git a/packages/rocketchat-lib/server/functions/setUsername.coffee b/packages/rocketchat-lib/server/functions/setUsername.coffee index a3afa4a49e9e..f07e63275384 100644 --- a/packages/rocketchat-lib/server/functions/setUsername.coffee +++ b/packages/rocketchat-lib/server/functions/setUsername.coffee @@ -61,13 +61,9 @@ RocketChat._setUsername = (userId, username) -> RocketChat.models.Subscriptions.setUserUsernameByUserId user._id, username RocketChat.models.Subscriptions.setNameForDirectRoomsWithOldName previousUsername, username - rs = RocketChatFileAvatarInstance.getFileWithReadStream(encodeURIComponent("#{previousUsername}.jpg")) - if rs? - RocketChatFileAvatarInstance.deleteFile encodeURIComponent("#{username}.jpg") - ws = RocketChatFileAvatarInstance.createWriteStream encodeURIComponent("#{username}.jpg"), rs.contentType - ws.on 'end', Meteor.bindEnvironment -> - RocketChatFileAvatarInstance.deleteFile encodeURIComponent("#{previousUsername}.jpg") - rs.readStream.pipe(ws) + fileStore = FileUpload.getStore('Avatars') + file = fileStore.model.findOneByName(previousUsername) + fileStore.model.updateFileNameById(file._id, username) # Set new username RocketChat.models.Users.setUsername user._id, username diff --git a/server/startup/avatar.js b/server/startup/avatar.js index ae69381aff5e..0fa118f685e1 100644 --- a/server/startup/avatar.js +++ b/server/startup/avatar.js @@ -1,39 +1,5 @@ /* globals FileUpload */ Meteor.startup(function() { - let storeType = 'GridFS'; - - if (RocketChat.settings.get('Accounts_AvatarStoreType')) { - storeType = RocketChat.settings.get('Accounts_AvatarStoreType'); - } - - const RocketChatStore = RocketChatFile[storeType]; - - if (!RocketChatStore) { - throw new Error(`Invalid RocketChatStore type [${ storeType }]`); - } - - console.log((`Using ${ storeType } for Avatar storage`).green); - - function transformWrite(file, readStream, writeStream) { - if (RocketChatFile.enabled === false || RocketChat.settings.get('Accounts_AvatarResize') !== true) { - return readStream.pipe(writeStream); - } - const height = RocketChat.settings.get('Accounts_AvatarSize'); - const width = height; - return RocketChatFile.gm(readStream, file.fileName).background('#ffffff').resize(width, `${ height }^`).gravity('Center').crop(width, height).extent(width, height).stream('jpeg').pipe(writeStream); - } - - let path = '~/uploads'; - if (RocketChat.settings.get('Accounts_AvatarStorePath') && RocketChat.settings.get('Accounts_AvatarStorePath').trim() !== '') { - path = RocketChat.settings.get('Accounts_AvatarStorePath'); - } - - this.RocketChatFileAvatarInstance = new RocketChatStore({ - name: 'avatars', - absolutePath: path, - transformWrite - }); - WebApp.connectHandlers.use('/avatar/', Meteor.bindEnvironment(function(req, res/*, next*/) { const params = { username: decodeURIComponent(req.url.replace(/^\//, '').replace(/\?.*$/, '')) diff --git a/server/startup/migrations/v002.js b/server/startup/migrations/v002.js index ad2d589a3cb8..e67336ed640d 100644 --- a/server/startup/migrations/v002.js +++ b/server/startup/migrations/v002.js @@ -26,13 +26,17 @@ RocketChat.Migrations.add({ const {image, contentType} = RocketChatFile.dataURIParse(dataURI); const rs = RocketChatFile.bufferToStream(new Buffer(image, 'base64')); - const ws = RocketChatFileAvatarInstance.createWriteStream(`${ user.username }.jpg`, contentType); + const fileStore = FileUpload.getStore('Avatars'); + fileStore.deleteByName(user.username); - ws.on('end', Meteor.bindEnvironment(function() { - return RocketChat.models.Users.setAvatarOrigin(user._id, service); - })); + const file = { + userId: user._id, + type: contentType + }; - return rs.pipe(ws); + fileStore.insert(file, rs, () => { + return RocketChat.models.Users.setAvatarOrigin(user._id, service); + }); }); } }); From 014241c352eaf0e694e0f869b76f63653cd31d20 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Tue, 2 May 2017 12:39:30 -0300 Subject: [PATCH 12/42] Make Amazon S3 work again --- .../client/lib/FileUploadAmazonS3.js | 3 +- .../server/config/configFileUploadAmazonS3.js | 48 +++++++++++-------- 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js b/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js index 0c6f3efd9054..6472ad7bbea1 100644 --- a/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js +++ b/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js @@ -7,6 +7,7 @@ FileUpload.AmazonS3 = class FileUploadAmazonS3 extends FileUploadBase { 'upload': 'rocketchat-uploads', 'avatar': 'rocketchat-avatars' }; + this.directive = directive; this.uploader = new Slingshot.Upload(directives[directive], meta); } @@ -23,7 +24,7 @@ FileUpload.AmazonS3 = class FileUploadAmazonS3 extends FileUploadBase { file._id = downloadUrl.substr(downloadUrl.lastIndexOf('/') + 1); file.url = downloadUrl; - return callback(null, file, 's3'); + return callback(null, file, this.directive === 'avatar' ? 'S3:Avatars' : 'S3:Uploads'); } }); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js index fe18ecaacebe..c4f3e13e0cfa 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js @@ -17,27 +17,37 @@ const generateURL = function(file) { return `${ file.url }?AWSAccessKeyId=${ encodeURIComponent(S3accessKey) }&Expires=${ expires }&Signature=${ encodeURIComponent(signature) }`; }; +const getFile = function(file, req, res) { + const fileUrl = generateURL(file); + + if (fileUrl) { + res.setHeader('Location', fileUrl); + res.writeHead(302); + } + res.end(); +}; + +const deleteFile = function(file) { + const s3 = new AWS.S3(); + const request = s3.deleteObject({ + Bucket: file.s3.bucket, + Key: file.s3.path + file._id + }); + request.send(); +}; + new FileUploadClass({ name: 'S3:Uploads', - get(file, req, res) { - const fileUrl = generateURL(file); + get: getFile, + delete: deleteFile +}); - if (fileUrl) { - res.setHeader('Location', fileUrl); - res.writeHead(302); - } - res.end(); - }, - - delete(file) { - const s3 = new AWS.S3(); - const request = s3.deleteObject({ - Bucket: file.s3.bucket, - Key: file.s3.path + file._id - }); - request.send(); - } +new FileUploadClass({ + name: 'S3:Avatars', + + get: getFile, + delete: deleteFile }); function createDirective(directiveName, { key, bucket, accessKey, secretKey, region, acl, cdn, bucketUrl}) { @@ -89,7 +99,7 @@ const configureSlingshot = _.debounce(() => { path } }; - const fileId = RocketChat.models.Uploads.insertFileInit(this.userId, 's3', file, upload); + const fileId = RocketChat.models.Uploads.insertFileInit(this.userId, 'S3:Uploads', file, upload); return path + fileId; } @@ -110,7 +120,7 @@ const configureSlingshot = _.debounce(() => { } }; delete file.name; - RocketChat.models.Avatars.insertAvatarFileInit(user.username, this.userId, 's3', file, upload); + RocketChat.models.Avatars.insertAvatarFileInit(user.username, this.userId, 'S3:Avatars', file, upload); return path + user.username; } From 451b055a70e375214cf723f6ef1d7392113b42a5 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Tue, 2 May 2017 19:51:45 -0300 Subject: [PATCH 13/42] Fix upload for Google Cloud Storage --- .../client/lib/FileUploadAmazonS3.js | 2 +- .../client/lib/FileUploadGoogleStorage.js | 2 +- .../server/config/configFileUploadAmazonS3.js | 8 +-- .../config/configFileUploadGoogleStorage.js | 60 +++++++++++-------- .../rocketchat-ui/client/lib/fileUpload.js | 2 +- 5 files changed, 42 insertions(+), 32 deletions(-) diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js b/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js index 6472ad7bbea1..678830d2bf53 100644 --- a/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js +++ b/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js @@ -24,7 +24,7 @@ FileUpload.AmazonS3 = class FileUploadAmazonS3 extends FileUploadBase { file._id = downloadUrl.substr(downloadUrl.lastIndexOf('/') + 1); file.url = downloadUrl; - return callback(null, file, this.directive === 'avatar' ? 'S3:Avatars' : 'S3:Uploads'); + return callback(null, file, this.directive === 'avatar' ? 'AmazonS3:Avatars' : 'AmazonS3:Uploads'); } }); diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorage.js b/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorage.js index b4b047c42b8b..a5aa21485b28 100644 --- a/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorage.js +++ b/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorage.js @@ -23,7 +23,7 @@ FileUpload.GoogleCloudStorage = class FileUploadGoogleCloudStorage extends FileU file._id = downloadUrl.substr(downloadUrl.lastIndexOf('/') + 1); file.url = downloadUrl; - return callback(null, file, 'googleCloudStorage'); + return callback(null, file, 'GoogleCloudStorage:Uploads'); } }); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js index c4f3e13e0cfa..b954f95e710b 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js @@ -37,14 +37,14 @@ const deleteFile = function(file) { }; new FileUploadClass({ - name: 'S3:Uploads', + name: 'AmazonS3:Uploads', get: getFile, delete: deleteFile }); new FileUploadClass({ - name: 'S3:Avatars', + name: 'AmazonS3:Avatars', get: getFile, delete: deleteFile @@ -99,7 +99,7 @@ const configureSlingshot = _.debounce(() => { path } }; - const fileId = RocketChat.models.Uploads.insertFileInit(this.userId, 'S3:Uploads', file, upload); + const fileId = RocketChat.models.Uploads.insertFileInit(this.userId, 'AmazonS3:Uploads', file, upload); return path + fileId; } @@ -120,7 +120,7 @@ const configureSlingshot = _.debounce(() => { } }; delete file.name; - RocketChat.models.Avatars.insertAvatarFileInit(user.username, this.userId, 'S3:Avatars', file, upload); + RocketChat.models.Avatars.insertAvatarFileInit(user.username, this.userId, 'AmazonS3:Avatars', file, upload); return path + user.username; } diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js index 69e0b514ba0c..c9cd841aacb8 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js @@ -63,36 +63,46 @@ function createDirective(directiveName, { key, bucket, accessId, secret }) { } } -new FileUploadClass({ - name: 'googleCloudStorage', +const getFile = function(file, req, res) { + const fileUrl = generateGetURL({ file }); - get(file, req, res) { - const fileUrl = generateGetURL({ file }); + if (fileUrl) { + res.setHeader('Location', fileUrl); + res.writeHead(302); + } + res.end(); +}; - if (fileUrl) { - res.setHeader('Location', fileUrl); - res.writeHead(302); - } - res.end(); - }, +const deleteFile = function(file) { + if (!file || !file.googleCloudStorage) { + console.warn('Failed to delete a file which is uploaded to Google Cloud Storage, the file and googleCloudStorage properties are not defined.'); + return; + } - delete(file) { - if (!file || !file.googleCloudStorage) { - console.warn('Failed to delete a file which is uploaded to Google Cloud Storage, the file and googleCloudStorage properties are not defined.'); - return; - } + // RocketChat.models.Uploads.deleteFile(file._id); + + const url = generateDeleteUrl({ file }); + + if (_.isEmpty(url)) { + console.warn('Failed to delete a file which is uploaded to Google Cloud Storage, failed to generate a delete url.'); + return; + } - // RocketChat.models.Uploads.deleteFile(file._id); + HTTP.call('DELETE', url); +}; - const url = generateDeleteUrl({ file }); +new FileUploadClass({ + name: 'GoogleCloudStorage:Uploads', - if (_.isEmpty(url)) { - console.warn('Failed to delete a file which is uploaded to Google Cloud Storage, failed to generate a delete url.'); - return; - } + get: getFile, + delete: deleteFile +}); - HTTP.call('DELETE', url); - } +new FileUploadClass({ + name: 'GoogleCloudStorage:Avatars', + + get: getFile, + delete: deleteFile }); const createGoogleStorageDirective = _.debounce(() => { @@ -108,7 +118,7 @@ const createGoogleStorageDirective = _.debounce(() => { path } }; - const fileId = RocketChat.models.Uploads.insertFileInit(this.userId, 'googleCloudStorage', file, upload); + const fileId = RocketChat.models.Uploads.insertFileInit(this.userId, 'GoogleCloudStorage:Uploads', file, upload); return path + fileId; } @@ -128,7 +138,7 @@ const createGoogleStorageDirective = _.debounce(() => { } }; delete file.name; - RocketChat.models.Avatars.insertAvatarFileInit(user.username, this.userId, 'googleCloudStorage', file, upload); + RocketChat.models.Avatars.insertAvatarFileInit(user.username, this.userId, 'GoogleCloudStorage:Avatars', file, upload); return path + user.username; } diff --git a/packages/rocketchat-ui/client/lib/fileUpload.js b/packages/rocketchat-ui/client/lib/fileUpload.js index 20d2007951c0..5f8cb982fab4 100644 --- a/packages/rocketchat-ui/client/lib/fileUpload.js +++ b/packages/rocketchat-ui/client/lib/fileUpload.js @@ -196,7 +196,7 @@ fileUpload = function(filesToUpload) { const uploading = Session.get('uploading'); if (uploading !== null) { const item = _.findWhere(uploading, { - id: this.id + id: upload.id }); return Session.set('uploading', _.without(uploading, item)); } From a9b89bdd0c1747d2d672b13b15dbc7ab1cc60ee7 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Mon, 22 May 2017 16:35:59 -0300 Subject: [PATCH 14/42] Add backwards compatibility --- lib/fileUpload.js | 3 +++ .../server/config/configFileUploadAmazonS3.js | 12 +++++++++++- .../server/config/configFileUploadFileSystem.js | 2 ++ .../server/config/configFileUploadGoogleStorage.js | 9 +++++++++ .../server/config/configFileUploadGridFS.js | 6 ------ .../rocketchat-file-upload/server/lib/FileUpload.js | 4 +++- 6 files changed, 28 insertions(+), 8 deletions(-) diff --git a/lib/fileUpload.js b/lib/fileUpload.js index f5eafb6e0985..ae75ffa40443 100644 --- a/lib/fileUpload.js +++ b/lib/fileUpload.js @@ -73,6 +73,9 @@ if (UploadFS) { } }); + // DEPRECATED: backwards compatibility (remove) + UploadFS.getStores()['rocketchat_uploads'] = UploadFS.getStores()['GridFS:Uploads']; + Meteor.fileStoreAvatar = new UploadFS.store.GridFS({ collection: RocketChat.models.Avatars.model, name: 'GridFS:Avatars', diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js index ebce1c102a61..594f7c5574c7 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js @@ -1,4 +1,5 @@ -/* globals Slingshot, FileUpload, AWS, FileUploadClass */ +/* globals Slingshot, FileUpload, AWS */ +import { FileUploadClass } from '../lib/FileUpload'; import AWS4 from '../lib/AWS4.js'; let S3accessKey; @@ -47,6 +48,15 @@ const deleteFile = function(file) { request.send(); }; +// DEPRECATED: backwards compatibility (remove) +new FileUploadClass({ + name: 's3', + model: 'Uploads', + + get: getFile, + delete: deleteFile +}); + new FileUploadClass({ name: 'AmazonS3:Uploads', diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js b/packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js index 9ffdb26a7a9c..680019a966ba 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js @@ -121,6 +121,8 @@ const createFileSystemStore = _.debounce(function() { transformWrite }); + UploadFS.getStores()['fileSystem'] = UploadFS.getStores()[FileSystemUploads.name]; + FileSystemAvatars.store = new UploadFS.store.Local({ path: RocketChat.settings.get('FileUpload_FileSystemPath'), //'/tmp/uploads/photos', collection: FileSystemAvatars.model.model, diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js index c9cd841aacb8..af41098cdc7a 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js @@ -91,6 +91,15 @@ const deleteFile = function(file) { HTTP.call('DELETE', url); }; +// DEPRECATED: backwards compatibility (remove) +new FileUploadClass({ + name: 'googleCloudStorage', + model: 'Uploads', + + get: getFile, + delete: deleteFile +}); + new FileUploadClass({ name: 'GoogleCloudStorage:Uploads', diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js b/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js index e26b7cc7d6b4..49d6b72e61b9 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js @@ -135,9 +135,6 @@ const insert = function(file, stream, cb) { new FileUploadClass({ name: 'GridFS:Uploads', - getStore() { - return Meteor.fileStore; - }, get(file, req, res) { file = FileUpload.addExtensionTo(file); @@ -155,9 +152,6 @@ new FileUploadClass({ new FileUploadClass({ name: 'GridFS:Avatars', - getStore() { - return Meteor.fileStoreAvatar; - }, get(file, req, res) { const reqModifiedHeader = req.headers['if-modified-since']; diff --git a/packages/rocketchat-file-upload/server/lib/FileUpload.js b/packages/rocketchat-file-upload/server/lib/FileUpload.js index f606d0ad0d4e..b3e616859c29 100644 --- a/packages/rocketchat-file-upload/server/lib/FileUpload.js +++ b/packages/rocketchat-file-upload/server/lib/FileUpload.js @@ -1,3 +1,5 @@ +/* globals UploadFS */ + import mime from 'mime-type/with-db'; Object.assign(FileUpload, { @@ -52,7 +54,7 @@ export class FileUploadClass { constructor({ name, model, store, get, insert, getStore }) { this.name = name; this.model = model || this.getModelFromName(); - this._store = store; + this._store = store || UploadFS.getStore(name); this.get = get; this.insert = insert; From 8cdeca6430fbdcd27eaaa8b079b555cb2428dfe8 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Wed, 24 May 2017 22:21:18 -0300 Subject: [PATCH 15/42] Add support to AmazonS3 Server for uploads --- package.json | 1 + .../client/lib/FileUploadAmazonS3Server.js | 56 ++++++++ packages/rocketchat-file-upload/package.js | 2 + .../config/configFileUploadAmazonS3Server.js | 114 +++++++++++++++ .../server/startup/settings.js | 3 + packages/rocketchat-file-upload/ufs/client.js | 6 + packages/rocketchat-file-upload/ufs/server.js | 132 ++++++++++++++++++ 7 files changed, 314 insertions(+) create mode 100644 packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3Server.js create mode 100644 packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3Server.js create mode 100644 packages/rocketchat-file-upload/ufs/client.js create mode 100644 packages/rocketchat-file-upload/ufs/server.js diff --git a/package.json b/package.json index 2dca607b9f1e..1b27d24f9c9e 100644 --- a/package.json +++ b/package.json @@ -78,6 +78,7 @@ "supertest": "^3.0.0" }, "dependencies": { + "aws-sdk": "^2.55.0", "babel-runtime": "^6.23.0", "bcrypt": "^1.0.2", "codemirror": "^5.25.2", diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3Server.js b/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3Server.js new file mode 100644 index 000000000000..b615c64331d4 --- /dev/null +++ b/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3Server.js @@ -0,0 +1,56 @@ +/* globals FileUpload, FileUploadBase, UploadFS, AmazonS3ServerStore:true, AmazonS3ServerStoreAvatar:true */ + +import '../../ufs/client.js'; + +AmazonS3ServerStore = new UploadFS.store.AmazonS3({ + collection: RocketChat.models.Uploads.model, + name: 'AmazonS3Server:Uploads', + filter: new UploadFS.Filter({ + onCheck: FileUpload.validateFileUpload + }) +}); + +AmazonS3ServerStoreAvatar = new UploadFS.store.AmazonS3({ + collection: RocketChat.models.Avatars.model, + name: 'AmazonS3Server:Avatars', + filter: new UploadFS.Filter({ + onCheck: FileUpload.validateFileUpload + }) +}); + +FileUpload.AmazonS3Server = class FileUploadAmazonS3Server extends FileUploadBase { + constructor(directive, meta, file) { + super(meta, file); + console.log('AmazonS3Server', {directive, meta, file}); + this.store = directive === 'avatar' ? AmazonS3ServerStoreAvatar : AmazonS3ServerStore; + } + + start(callback) { + this.handler = new UploadFS.Uploader({ + store: this.store, + data: this.file, + file: this.meta, + onError: (err) => { + return callback(err); + }, + onComplete: (fileData) => { + const file = _.pick(fileData, '_id', 'type', 'size', 'name', 'identify', 'description'); + + file.url = fileData.url.replace(Meteor.absoluteUrl(), '/'); + return callback(null, file, 'fs'); + } + }); + + this.handler.onProgress = (file, progress) => { + this.onProgress(progress); + }; + + return this.handler.start(); + } + + onProgress() {} + + stop() { + return this.handler.stop(); + } +}; diff --git a/packages/rocketchat-file-upload/package.js b/packages/rocketchat-file-upload/package.js index 8ba5db02407d..1587984f7b6b 100644 --- a/packages/rocketchat-file-upload/package.js +++ b/packages/rocketchat-file-upload/package.js @@ -28,6 +28,7 @@ Package.onUse(function(api) { api.addFiles('lib/FileUploadBase.js'); api.addFiles('client/lib/FileUploadAmazonS3.js', 'client'); + api.addFiles('client/lib/FileUploadAmazonS3Server.js', 'client'); api.addFiles('client/lib/FileUploadFileSystem.js', 'client'); api.addFiles('client/lib/FileUploadGoogleStorage.js', 'client'); api.addFiles('client/lib/FileUploadGridFS.js', 'client'); @@ -37,6 +38,7 @@ Package.onUse(function(api) { api.addFiles('server/lib/requests.js', 'server'); api.addFiles('server/config/configFileUploadAmazonS3.js', 'server'); + api.addFiles('server/config/configFileUploadAmazonS3Server.js', 'server'); api.addFiles('server/config/configFileUploadFileSystem.js', 'server'); api.addFiles('server/config/configFileUploadGoogleStorage.js', 'server'); api.addFiles('server/config/configFileUploadGridFS.js', 'server'); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3Server.js b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3Server.js new file mode 100644 index 000000000000..9e4a8cae1e70 --- /dev/null +++ b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3Server.js @@ -0,0 +1,114 @@ +/* globals FileUpload, UploadFS, RocketChatFile */ + +import fs from 'fs'; +import { FileUploadClass } from '../lib/FileUpload'; +import '../../ufs/server.js'; + +const Future = Npm.require('fibers/future'); + +const insert = function(file, stream, cb) { + const fileId = this.store.create(file); + + this.store.write(stream, fileId, cb); +}; + +const AmazonS3ServerUploads = new FileUploadClass({ + name: 'AmazonS3Server:Uploads', + // store setted bellow + + get(file, req, res) { + const fileUrl = AmazonS3ServerUploads.store.getS3URL(file); + + if (fileUrl) { + res.setHeader('Location', fileUrl); + res.writeHead(302); + } + res.end(); + }, + + insert +}); + +const onValidate = function(file) { + if (RocketChatFile.enabled === false || !/^image\/((x-windows-)?bmp|p?jpeg|png)$/.test(file.type)) { + return; + } + + const tmpFile = UploadFS.getTempFilePath(file._id); + + const fut = new Future(); + + const identify = Meteor.bindEnvironment((err, data) => { + if (err != null) { + console.error(err); + return fut.return(); + } + + file.identify = { + format: data.format, + size: data.size + }; + + if ([null, undefined, '', 'Unknown', 'Undefined'].includes(data.Orientation)) { + return fut.return(); + } + + RocketChatFile.gm(tmpFile).autoOrient().write(tmpFile, Meteor.bindEnvironment((err) => { + if (err != null) { + console.error(err); + } + + const size = fs.lstatSync(tmpFile).size; + this.getCollection().direct.update({_id: file._id}, {$set: {size}}); + fut.return(); + })); + }); + + RocketChatFile.gm(tmpFile).identify(identify); + + return fut.wait(); +}; + +const configure = _.debounce(function() { + const stores = UploadFS.getStores(); + delete stores[AmazonS3ServerUploads.name]; + + const Bucket = RocketChat.settings.get('FileUpload_S3_Bucket'); + const Acl = RocketChat.settings.get('FileUpload_S3_Acl'); + const AWSAccessKeyId = RocketChat.settings.get('FileUpload_S3_AWSAccessKeyId'); + const AWSSecretAccessKey = RocketChat.settings.get('FileUpload_S3_AWSSecretAccessKey'); + const URLExpiryTimeSpan = RocketChat.settings.get('FileUpload_S3_URLExpiryTimeSpan'); + const Region = RocketChat.settings.get('FileUpload_S3_Region'); + // const CDN = RocketChat.settings.get('FileUpload_S3_CDN'); + // const BucketURL = RocketChat.settings.get('FileUpload_S3_BucketURL'); + + AmazonS3ServerUploads.store = new UploadFS.store.AmazonS3({ + connection: { + accessKeyId: AWSAccessKeyId, + secretAccessKey: AWSSecretAccessKey, + signatureVersion: 'v4', + params: { + Bucket, + ACL: Acl + }, + region: Region + }, + URLExpiryTimeSpan, + collection: AmazonS3ServerUploads.model.model, + filter: new UploadFS.Filter({ + onCheck: FileUpload.validateFileUpload + }), + name: AmazonS3ServerUploads.name, + onFinishUpload(file) { + AmazonS3ServerUploads.model.update({_id: file._id}, { + $set: { + s3: file.s3 + } + }); + }, + onValidate + }); + +}, 500); + +RocketChat.settings.get(/^FileUpload_S3_/, configure); diff --git a/packages/rocketchat-file-upload/server/startup/settings.js b/packages/rocketchat-file-upload/server/startup/settings.js index fd1cea0b15a6..c49b58cd07ff 100644 --- a/packages/rocketchat-file-upload/server/startup/settings.js +++ b/packages/rocketchat-file-upload/server/startup/settings.js @@ -29,6 +29,9 @@ RocketChat.settings.addGroup('FileUpload', function() { }, { key: 'AmazonS3', i18nLabel: 'AmazonS3' + }, { + key: 'AmazonS3Server', + i18nLabel: 'AmazonS3Server' }, { key: 'GoogleCloudStorage', i18nLabel: 'GoogleCloudStorage' diff --git a/packages/rocketchat-file-upload/ufs/client.js b/packages/rocketchat-file-upload/ufs/client.js new file mode 100644 index 000000000000..6d9f66f467a4 --- /dev/null +++ b/packages/rocketchat-file-upload/ufs/client.js @@ -0,0 +1,6 @@ +import {UploadFS} from 'meteor/jalik:ufs'; + +export class AmazonS3Store extends UploadFS.Store {} + +// Add store to UFS namespace +UploadFS.store.AmazonS3 = AmazonS3Store; diff --git a/packages/rocketchat-file-upload/ufs/server.js b/packages/rocketchat-file-upload/ufs/server.js new file mode 100644 index 000000000000..c57e7da0c082 --- /dev/null +++ b/packages/rocketchat-file-upload/ufs/server.js @@ -0,0 +1,132 @@ +import {_} from 'meteor/underscore'; +// import {check} from 'meteor/check'; +import {Meteor} from 'meteor/meteor'; +import {UploadFS} from 'meteor/jalik:ufs'; +import S3 from 'aws-sdk/clients/s3'; +import stream from 'stream'; + +/** + * GridFS store + * @param options + * @constructor + */ +export class AmazonS3Store extends UploadFS.Store { + + constructor(options) { + // Default options + // options.secretAccessKey, + // options.accessKeyId, + // options.region, + // options.sslEnabled // optional + + options = _.extend({ + httpOptions: { + timeout: 6000, + agent: false + } + }, options); + + super(options); + + const classOptions = options; + + if (Meteor.isServer) { + const s3 = new S3(options.connection); + + this.getPath = function(file) { + return `${ RocketChat.hostname }/${ file.rid }/${ file.userId }/${ file._id }`; + }; + + this.getS3URL = function(file) { + const params = { + Key: this.getPath(file), + Expires: classOptions.URLExpiryTimeSpan + }; + + return s3.getSignedUrl('getObject', params); + }; + + /** + * Removes the file + * @param fileId + * @param callback + */ + this.delete = function(fileId, callback) { + const file = this.getCollection().findOne({_id: fileId}); + const params = { + Key: this.getPath(file) + }; + + s3.deleteObject(params, (err, data) => { + if (err) { + console.error(err); + } + + callback && callback(err, data); + }); + }; + + /** + * Returns the file read stream + * @param fileId + * @param file + * @param options + * @return {*} + */ + this.getReadStream = function(fileId, file, options = {}) { + const params = { + Key: this.getPath(file) + }; + + if (options.start && options.end) { + params.Range = `${ options.start } - ${ options.end }`; + } + + return s3.getObject(params).createReadStream(); + }; + + /** + * Returns the file write stream + * @param fileId + * @param file + * @param options + * @return {*} + */ + this.getWriteStream = function(fileId, file/*, options*/) { + const writeStream = new stream.PassThrough(); + writeStream.length = file.size; + + writeStream.on('newListener', (event, listener) => { + if (event === 'finish') { + process.nextTick(() => { + writeStream.removeListener(event, listener); + writeStream.on('real_finish', listener); + }); + } + }); + + file.s3 = { + Key: this.getPath(file) + }; + + s3.putObject({ + Key: file.s3.Key, + Body: writeStream, + ContentType: file.type + + }, (error) => { + if (error) { + console.error(error); + } + + writeStream.emit('real_finish'); + }); + + return writeStream; + }; + } + } +} + +// Add store to UFS namespace +UploadFS.store.AmazonS3 = AmazonS3Store; From e02d5dcf92894f4d135d11b5621b7c5e6445a5d3 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Wed, 24 May 2017 22:50:17 -0300 Subject: [PATCH 16/42] Implement AmazonS3 Server for Avatars --- .../config/configFileUploadAmazonS3Server.js | 81 +++++++++++++++---- packages/rocketchat-file-upload/ufs/server.js | 6 +- 2 files changed, 66 insertions(+), 21 deletions(-) diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3Server.js b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3Server.js index 9e4a8cae1e70..4d0b9a421c91 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3Server.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3Server.js @@ -12,20 +12,29 @@ const insert = function(file, stream, cb) { this.store.write(stream, fileId, cb); }; +const get = function(file, req, res) { + const fileUrl = this.store.getS3URL(file); + + if (fileUrl) { + res.setHeader('Location', fileUrl); + res.writeHead(302); + } + res.end(); +}; + const AmazonS3ServerUploads = new FileUploadClass({ name: 'AmazonS3Server:Uploads', // store setted bellow - get(file, req, res) { - const fileUrl = AmazonS3ServerUploads.store.getS3URL(file); + get, + insert +}); - if (fileUrl) { - res.setHeader('Location', fileUrl); - res.writeHead(302); - } - res.end(); - }, +const AmazonS3ServerAvatars = new FileUploadClass({ + name: 'AmazonS3Server:Avatars', + // store setted bellow + get, insert }); @@ -72,6 +81,7 @@ const onValidate = function(file) { const configure = _.debounce(function() { const stores = UploadFS.getStores(); delete stores[AmazonS3ServerUploads.name]; + delete stores[AmazonS3ServerAvatars.name]; const Bucket = RocketChat.settings.get('FileUpload_S3_Bucket'); const Acl = RocketChat.settings.get('FileUpload_S3_Acl'); @@ -82,7 +92,7 @@ const configure = _.debounce(function() { // const CDN = RocketChat.settings.get('FileUpload_S3_CDN'); // const BucketURL = RocketChat.settings.get('FileUpload_S3_BucketURL'); - AmazonS3ServerUploads.store = new UploadFS.store.AmazonS3({ + const config = { connection: { accessKeyId: AWSAccessKeyId, secretAccessKey: AWSSecretAccessKey, @@ -93,21 +103,60 @@ const configure = _.debounce(function() { }, region: Region }, - URLExpiryTimeSpan, + URLExpiryTimeSpan + }; + + AmazonS3ServerUploads.store = new UploadFS.store.AmazonS3(Object.assign({ collection: AmazonS3ServerUploads.model.model, filter: new UploadFS.Filter({ onCheck: FileUpload.validateFileUpload }), name: AmazonS3ServerUploads.name, + onValidate + }, config)); + + AmazonS3ServerAvatars.store = new UploadFS.store.AmazonS3(Object.assign({ + collection: AmazonS3ServerAvatars.model.model, + name: AmazonS3ServerAvatars.name, onFinishUpload(file) { - AmazonS3ServerUploads.model.update({_id: file._id}, { - $set: { - s3: file.s3 + // update file record to match user's username + const user = RocketChat.models.Users.findOneById(file.userId); + const oldAvatar = AmazonS3ServerAvatars.model.findOneByName(user.username); + if (oldAvatar) { + try { + AmazonS3ServerAvatars.deleteById(oldAvatar._id); + } catch (e) { + console.error(e); } - }); + } + AmazonS3ServerAvatars.model.updateFileNameById(file._id, user.username); + // console.log('upload finished ->', file); }, - onValidate - }); + onValidate(file) { + if (RocketChatFile.enabled === false || RocketChat.settings.get('Accounts_AvatarResize') !== true) { + return; + } + + const tmpFile = UploadFS.getTempFilePath(file._id); + + const fut = new Future(); + + const height = RocketChat.settings.get('Accounts_AvatarSize'); + const width = height; + + RocketChatFile.gm(tmpFile).background('#ffffff').resize(width, `${ height }^`).gravity('Center').crop(width, height).extent(width, height).setFormat('jpeg').write(tmpFile, Meteor.bindEnvironment((err) => { + if (err != null) { + console.error(err); + } + + const size = fs.lstatSync(tmpFile).size; + this.getCollection().direct.update({_id: file._id}, {$set: {size}}); + fut.return(); + })); + + return fut.wait(); + } + }, config)); }, 500); diff --git a/packages/rocketchat-file-upload/ufs/server.js b/packages/rocketchat-file-upload/ufs/server.js index c57e7da0c082..df53aca2d000 100644 --- a/packages/rocketchat-file-upload/ufs/server.js +++ b/packages/rocketchat-file-upload/ufs/server.js @@ -105,12 +105,8 @@ export class AmazonS3Store extends UploadFS.Store { } }); - file.s3 = { - Key: this.getPath(file) - }; - s3.putObject({ - Key: file.s3.Key, + Key: this.getPath(file), Body: writeStream, ContentType: file.type From d2e19244b20b42863141edd976f0f7fa33bc9a1f Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Thu, 25 May 2017 17:08:27 -0300 Subject: [PATCH 17/42] Add Google Cloud Storage Server option --- package.json | 1 + .../client/lib/FileUploadAmazonS3Server.js | 2 +- .../lib/FileUploadGoogleStorageServer.js | 56 ++++++ packages/rocketchat-file-upload/package.js | 2 + .../config/configFileUploadAmazonS3Server.js | 2 +- .../configFileUploadGoogleStorageServer.js | 161 ++++++++++++++++++ .../server/startup/settings.js | 3 + .../ufs/{ => AmazonS3}/client.js | 0 .../ufs/{ => AmazonS3}/server.js | 3 +- .../ufs/GoogleStorage/client.js | 6 + .../ufs/GoogleStorage/server.js | 87 ++++++++++ 11 files changed, 319 insertions(+), 4 deletions(-) create mode 100644 packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorageServer.js create mode 100644 packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorageServer.js rename packages/rocketchat-file-upload/ufs/{ => AmazonS3}/client.js (100%) rename packages/rocketchat-file-upload/ufs/{ => AmazonS3}/server.js (97%) create mode 100644 packages/rocketchat-file-upload/ufs/GoogleStorage/client.js create mode 100644 packages/rocketchat-file-upload/ufs/GoogleStorage/server.js diff --git a/package.json b/package.json index 1b27d24f9c9e..2310d3c47532 100644 --- a/package.json +++ b/package.json @@ -78,6 +78,7 @@ "supertest": "^3.0.0" }, "dependencies": { + "@google-cloud/storage": "^1.1.1", "aws-sdk": "^2.55.0", "babel-runtime": "^6.23.0", "bcrypt": "^1.0.2", diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3Server.js b/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3Server.js index b615c64331d4..ed58adafa345 100644 --- a/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3Server.js +++ b/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3Server.js @@ -1,6 +1,6 @@ /* globals FileUpload, FileUploadBase, UploadFS, AmazonS3ServerStore:true, AmazonS3ServerStoreAvatar:true */ -import '../../ufs/client.js'; +import '../../ufs/AmazonS3/client.js'; AmazonS3ServerStore = new UploadFS.store.AmazonS3({ collection: RocketChat.models.Uploads.model, diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorageServer.js b/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorageServer.js new file mode 100644 index 000000000000..9e64169382b9 --- /dev/null +++ b/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorageServer.js @@ -0,0 +1,56 @@ +/* globals FileUpload, FileUploadBase, UploadFS, GoogleCloudStorageServerStore:true, GoogleCloudStorageServerStoreAvatar:true */ + +import '../../ufs/GoogleStorage/client.js'; + +GoogleCloudStorageServerStore = new UploadFS.store.GoogleStorage({ + collection: RocketChat.models.Uploads.model, + name: 'GoogleCloudStorageServer:Uploads', + filter: new UploadFS.Filter({ + onCheck: FileUpload.validateFileUpload + }) +}); + +GoogleCloudStorageServerStoreAvatar = new UploadFS.store.GoogleStorage({ + collection: RocketChat.models.Avatars.model, + name: 'GoogleCloudStorageServer:Avatars', + filter: new UploadFS.Filter({ + onCheck: FileUpload.validateFileUpload + }) +}); + +FileUpload.GoogleCloudStorageServer = class FileUploadGoogleCloudStorageServer extends FileUploadBase { + constructor(directive, meta, file) { + super(meta, file); + console.log('GoogleCloudStorageServer', {directive, meta, file}); + this.store = directive === 'avatar' ? GoogleCloudStorageServerStoreAvatar : GoogleCloudStorageServerStore; + } + + start(callback) { + this.handler = new UploadFS.Uploader({ + store: this.store, + data: this.file, + file: this.meta, + onError: (err) => { + return callback(err); + }, + onComplete: (fileData) => { + const file = _.pick(fileData, '_id', 'type', 'size', 'name', 'identify', 'description'); + + file.url = fileData.url.replace(Meteor.absoluteUrl(), '/'); + return callback(null, file, 'fs'); + } + }); + + this.handler.onProgress = (file, progress) => { + this.onProgress(progress); + }; + + return this.handler.start(); + } + + onProgress() {} + + stop() { + return this.handler.stop(); + } +}; diff --git a/packages/rocketchat-file-upload/package.js b/packages/rocketchat-file-upload/package.js index 1587984f7b6b..a847f22070d5 100644 --- a/packages/rocketchat-file-upload/package.js +++ b/packages/rocketchat-file-upload/package.js @@ -31,6 +31,7 @@ Package.onUse(function(api) { api.addFiles('client/lib/FileUploadAmazonS3Server.js', 'client'); api.addFiles('client/lib/FileUploadFileSystem.js', 'client'); api.addFiles('client/lib/FileUploadGoogleStorage.js', 'client'); + api.addFiles('client/lib/FileUploadGoogleStorageServer.js', 'client'); api.addFiles('client/lib/FileUploadGridFS.js', 'client'); api.addFiles('client/lib/fileUploadHandler.js', 'client'); @@ -41,6 +42,7 @@ Package.onUse(function(api) { api.addFiles('server/config/configFileUploadAmazonS3Server.js', 'server'); api.addFiles('server/config/configFileUploadFileSystem.js', 'server'); api.addFiles('server/config/configFileUploadGoogleStorage.js', 'server'); + api.addFiles('server/config/configFileUploadGoogleStorageServer.js', 'server'); api.addFiles('server/config/configFileUploadGridFS.js', 'server'); api.addFiles('server/methods/sendFileMessage.js', 'server'); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3Server.js b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3Server.js index 4d0b9a421c91..35bf3ae37392 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3Server.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3Server.js @@ -2,7 +2,7 @@ import fs from 'fs'; import { FileUploadClass } from '../lib/FileUpload'; -import '../../ufs/server.js'; +import '../../ufs/AmazonS3/server.js'; const Future = Npm.require('fibers/future'); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorageServer.js b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorageServer.js new file mode 100644 index 000000000000..d51fb4bbbe36 --- /dev/null +++ b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorageServer.js @@ -0,0 +1,161 @@ +/* globals FileUpload, UploadFS, RocketChatFile */ + +import fs from 'fs'; +import { FileUploadClass } from '../lib/FileUpload'; +import '../../ufs/GoogleStorage/server.js'; + +const Future = Npm.require('fibers/future'); + +const insert = function(file, stream, cb) { + const fileId = this.store.create(file); + + this.store.write(stream, fileId, cb); +}; + +const get = function(file, req, res) { + this.store.getRedirectURL(file, (err, fileUrl) => { + if (err) { + console.error(err); + } + + if (fileUrl) { + res.setHeader('Location', fileUrl); + res.writeHead(302); + } + res.end(); + }); +}; + +const GoogleCloudStorageServerUploads = new FileUploadClass({ + name: 'GoogleCloudStorageServer:Uploads', + // store setted bellow + + get, + insert +}); + +const GoogleCloudStorageServerAvatars = new FileUploadClass({ + name: 'GoogleCloudStorageServer:Avatars', + // store setted bellow + + get, + insert +}); + +const onValidate = function(file) { + if (RocketChatFile.enabled === false || !/^image\/((x-windows-)?bmp|p?jpeg|png)$/.test(file.type)) { + return; + } + + const tmpFile = UploadFS.getTempFilePath(file._id); + + const fut = new Future(); + + const identify = Meteor.bindEnvironment((err, data) => { + if (err != null) { + console.error(err); + return fut.return(); + } + + file.identify = { + format: data.format, + size: data.size + }; + + if ([null, undefined, '', 'Unknown', 'Undefined'].includes(data.Orientation)) { + return fut.return(); + } + + RocketChatFile.gm(tmpFile).autoOrient().write(tmpFile, Meteor.bindEnvironment((err) => { + if (err != null) { + console.error(err); + } + + const size = fs.lstatSync(tmpFile).size; + this.getCollection().direct.update({_id: file._id}, {$set: {size}}); + fut.return(); + })); + }); + + RocketChatFile.gm(tmpFile).identify(identify); + + return fut.wait(); +}; + +const configure = _.debounce(function() { + const stores = UploadFS.getStores(); + delete stores[GoogleCloudStorageServerUploads.name]; + delete stores[GoogleCloudStorageServerAvatars.name]; + + // const type = RocketChat.settings.get('FileUpload_Storage_Type'); + const bucket = RocketChat.settings.get('FileUpload_GoogleStorage_Bucket'); + const accessId = RocketChat.settings.get('FileUpload_GoogleStorage_AccessId'); + const secret = RocketChat.settings.get('FileUpload_GoogleStorage_Secret'); + const URLExpiryTimeSpan = RocketChat.settings.get('FileUpload_S3_URLExpiryTimeSpan'); + + const config = { + connection: { + credentials: { + client_email: accessId, + private_key: secret + } + }, + bucket, + URLExpiryTimeSpan + }; + + GoogleCloudStorageServerUploads.store = new UploadFS.store.GoogleStorage(Object.assign({ + collection: GoogleCloudStorageServerUploads.model.model, + filter: new UploadFS.Filter({ + onCheck: FileUpload.validateFileUpload + }), + name: GoogleCloudStorageServerUploads.name, + onValidate + }, config)); + + GoogleCloudStorageServerAvatars.store = new UploadFS.store.GoogleStorage(Object.assign({ + collection: GoogleCloudStorageServerAvatars.model.model, + name: GoogleCloudStorageServerAvatars.name, + onFinishUpload(file) { + // update file record to match user's username + const user = RocketChat.models.Users.findOneById(file.userId); + const oldAvatar = GoogleCloudStorageServerAvatars.model.findOneByName(user.username); + if (oldAvatar) { + try { + GoogleCloudStorageServerAvatars.deleteById(oldAvatar._id); + } catch (e) { + console.error(e); + } + } + GoogleCloudStorageServerAvatars.model.updateFileNameById(file._id, user.username); + // console.log('upload finished ->', file); + }, + onValidate(file) { + if (RocketChatFile.enabled === false || RocketChat.settings.get('Accounts_AvatarResize') !== true) { + return; + } + + const tmpFile = UploadFS.getTempFilePath(file._id); + + const fut = new Future(); + + const height = RocketChat.settings.get('Accounts_AvatarSize'); + const width = height; + + RocketChatFile.gm(tmpFile).background('#ffffff').resize(width, `${ height }^`).gravity('Center').crop(width, height).extent(width, height).setFormat('jpeg').write(tmpFile, Meteor.bindEnvironment((err) => { + if (err != null) { + console.error(err); + } + + const size = fs.lstatSync(tmpFile).size; + this.getCollection().direct.update({_id: file._id}, {$set: {size}}); + fut.return(); + })); + + return fut.wait(); + } + }, config)); + +}, 500); + +RocketChat.settings.get(/^FileUpload_GoogleStorage_/, configure); diff --git a/packages/rocketchat-file-upload/server/startup/settings.js b/packages/rocketchat-file-upload/server/startup/settings.js index c49b58cd07ff..1431930a7e7c 100644 --- a/packages/rocketchat-file-upload/server/startup/settings.js +++ b/packages/rocketchat-file-upload/server/startup/settings.js @@ -35,6 +35,9 @@ RocketChat.settings.addGroup('FileUpload', function() { }, { key: 'GoogleCloudStorage', i18nLabel: 'GoogleCloudStorage' + }, { + key: 'GoogleCloudStorageServer', + i18nLabel: 'GoogleCloudStorageServer' }, { key: 'FileSystem', i18nLabel: 'FileSystem' diff --git a/packages/rocketchat-file-upload/ufs/client.js b/packages/rocketchat-file-upload/ufs/AmazonS3/client.js similarity index 100% rename from packages/rocketchat-file-upload/ufs/client.js rename to packages/rocketchat-file-upload/ufs/AmazonS3/client.js diff --git a/packages/rocketchat-file-upload/ufs/server.js b/packages/rocketchat-file-upload/ufs/AmazonS3/server.js similarity index 97% rename from packages/rocketchat-file-upload/ufs/server.js rename to packages/rocketchat-file-upload/ufs/AmazonS3/server.js index df53aca2d000..6dbbd4412cd4 100644 --- a/packages/rocketchat-file-upload/ufs/server.js +++ b/packages/rocketchat-file-upload/ufs/AmazonS3/server.js @@ -1,12 +1,11 @@ import {_} from 'meteor/underscore'; -// import {check} from 'meteor/check'; import {Meteor} from 'meteor/meteor'; import {UploadFS} from 'meteor/jalik:ufs'; import S3 from 'aws-sdk/clients/s3'; import stream from 'stream'; /** - * GridFS store + * AmazonS3 store * @param options * @constructor */ diff --git a/packages/rocketchat-file-upload/ufs/GoogleStorage/client.js b/packages/rocketchat-file-upload/ufs/GoogleStorage/client.js new file mode 100644 index 000000000000..2c11dc0d6828 --- /dev/null +++ b/packages/rocketchat-file-upload/ufs/GoogleStorage/client.js @@ -0,0 +1,6 @@ +import {UploadFS} from 'meteor/jalik:ufs'; + +export class GoogleStorageStore extends UploadFS.Store {} + +// Add store to UFS namespace +UploadFS.store.GoogleStorage = GoogleStorageStore; diff --git a/packages/rocketchat-file-upload/ufs/GoogleStorage/server.js b/packages/rocketchat-file-upload/ufs/GoogleStorage/server.js new file mode 100644 index 000000000000..f2cef3e11616 --- /dev/null +++ b/packages/rocketchat-file-upload/ufs/GoogleStorage/server.js @@ -0,0 +1,87 @@ +import {Meteor} from 'meteor/meteor'; +import {UploadFS} from 'meteor/jalik:ufs'; +import gcStorage from '@google-cloud/storage'; + +/** + * GoogleStorage store + * @param options + * @constructor + */ +export class GoogleStorageStore extends UploadFS.Store { + + constructor(options) { + super(options); + + const classOptions = options; + + if (Meteor.isServer) { + const gcs = gcStorage(options.connection); + const bucket = gcs.bucket(options.bucket); + + this.getPath = function(file) { + return `${ RocketChat.hostname }/${ file.rid }/${ file.userId }/${ file._id }`; + }; + + this.getRedirectURL = function(file, callback) { + const params = { + action: 'read', + responseDisposition: 'inline', + expires: Date.now()+classOptions.URLExpiryTimeSpan*1000 + }; + + bucket.file(this.getPath(file)).getSignedUrl(params, callback); + }; + + /** + * Removes the file + * @param fileId + * @param callback + */ + this.delete = function(fileId, callback) { + const file = this.getCollection().findOne({_id: fileId}); + bucket.file(this.getPath(file)).delete(function(err, data) { + if (err) { + console.error(err); + } + + callback && callback(err, data); + }); + }; + + /** + * Returns the file read stream + * @param fileId + * @param file + * @param options + * @return {*} + */ + this.getReadStream = function(fileId, file/*, options = {}*/) { + // TODO range? + return bucket.file(this.getPath(file)).createReadStream(); + }; + + /** + * Returns the file write stream + * @param fileId + * @param file + * @param options + * @return {*} + */ + this.getWriteStream = function(fileId, file/*, options*/) { + return bucket.file(this.getPath(file)).createWriteStream({ + gzip: false, + metadata: { + contentType: file.type, + contentDisposition: `inline; filename=${ file.name }` + // metadata: { + // custom: 'metadata' + // } + } + }); + }; + } + } +} + +// Add store to UFS namespace +UploadFS.store.GoogleStorage = GoogleStorageStore; From a8519bafb9cf84cd7fc9887f1459a6d52d90d083 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Thu, 25 May 2017 17:18:00 -0300 Subject: [PATCH 18/42] Remove use of slingshot from client --- .../client/lib/FileUploadAmazonS3.js | 65 ++-- .../client/lib/FileUploadAmazonS3Server.js | 56 ---- .../client/lib/FileUploadGoogleStorage.js | 64 ++-- .../lib/FileUploadGoogleStorageServer.js | 56 ---- packages/rocketchat-file-upload/package.js | 6 +- .../server/config/configFileUploadAmazonS3.js | 286 ++++++++---------- .../config/configFileUploadAmazonS3Server.js | 163 ---------- .../configFileUploadAmazonS3_Deprecated.js | 189 ++++++++++++ .../config/configFileUploadGoogleStorage.js | 268 ++++++++-------- .../configFileUploadGoogleStorageServer.js | 161 ---------- ...onfigFileUploadGoogleStorage_Deprecated.js | 165 ++++++++++ .../server/startup/settings.js | 6 - 12 files changed, 682 insertions(+), 803 deletions(-) delete mode 100644 packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3Server.js delete mode 100644 packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorageServer.js delete mode 100644 packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3Server.js create mode 100644 packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3_Deprecated.js delete mode 100644 packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorageServer.js create mode 100644 packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage_Deprecated.js diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js b/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js index 678830d2bf53..ed58adafa345 100644 --- a/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js +++ b/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js @@ -1,43 +1,56 @@ -/* globals FileUpload, FileUploadBase, Slingshot */ +/* globals FileUpload, FileUploadBase, UploadFS, AmazonS3ServerStore:true, AmazonS3ServerStoreAvatar:true */ -FileUpload.AmazonS3 = class FileUploadAmazonS3 extends FileUploadBase { +import '../../ufs/AmazonS3/client.js'; + +AmazonS3ServerStore = new UploadFS.store.AmazonS3({ + collection: RocketChat.models.Uploads.model, + name: 'AmazonS3Server:Uploads', + filter: new UploadFS.Filter({ + onCheck: FileUpload.validateFileUpload + }) +}); + +AmazonS3ServerStoreAvatar = new UploadFS.store.AmazonS3({ + collection: RocketChat.models.Avatars.model, + name: 'AmazonS3Server:Avatars', + filter: new UploadFS.Filter({ + onCheck: FileUpload.validateFileUpload + }) +}); + +FileUpload.AmazonS3Server = class FileUploadAmazonS3Server extends FileUploadBase { constructor(directive, meta, file) { super(meta, file); - const directives = { - 'upload': 'rocketchat-uploads', - 'avatar': 'rocketchat-avatars' - }; - this.directive = directive; - this.uploader = new Slingshot.Upload(directives[directive], meta); + console.log('AmazonS3Server', {directive, meta, file}); + this.store = directive === 'avatar' ? AmazonS3ServerStoreAvatar : AmazonS3ServerStore; } start(callback) { - this.uploader.send(this.file, (error, downloadUrl) => { - if (this.computation) { - this.computation.stop(); - } + this.handler = new UploadFS.Uploader({ + store: this.store, + data: this.file, + file: this.meta, + onError: (err) => { + return callback(err); + }, + onComplete: (fileData) => { + const file = _.pick(fileData, '_id', 'type', 'size', 'name', 'identify', 'description'); - if (error) { - return callback.call(this, error); - } else { - const file = _.pick(this.meta, 'type', 'size', 'name', 'identify', 'description'); - file._id = downloadUrl.substr(downloadUrl.lastIndexOf('/') + 1); - file.url = downloadUrl; - - return callback(null, file, this.directive === 'avatar' ? 'AmazonS3:Avatars' : 'AmazonS3:Uploads'); + file.url = fileData.url.replace(Meteor.absoluteUrl(), '/'); + return callback(null, file, 'fs'); } }); - this.computation = Tracker.autorun(() => { - this.onProgress(this.uploader.progress()); - }); + this.handler.onProgress = (file, progress) => { + this.onProgress(progress); + }; + + return this.handler.start(); } onProgress() {} stop() { - if (this.uploader && this.uploader.xhr) { - this.uploader.xhr.abort(); - } + return this.handler.stop(); } }; diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3Server.js b/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3Server.js deleted file mode 100644 index ed58adafa345..000000000000 --- a/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3Server.js +++ /dev/null @@ -1,56 +0,0 @@ -/* globals FileUpload, FileUploadBase, UploadFS, AmazonS3ServerStore:true, AmazonS3ServerStoreAvatar:true */ - -import '../../ufs/AmazonS3/client.js'; - -AmazonS3ServerStore = new UploadFS.store.AmazonS3({ - collection: RocketChat.models.Uploads.model, - name: 'AmazonS3Server:Uploads', - filter: new UploadFS.Filter({ - onCheck: FileUpload.validateFileUpload - }) -}); - -AmazonS3ServerStoreAvatar = new UploadFS.store.AmazonS3({ - collection: RocketChat.models.Avatars.model, - name: 'AmazonS3Server:Avatars', - filter: new UploadFS.Filter({ - onCheck: FileUpload.validateFileUpload - }) -}); - -FileUpload.AmazonS3Server = class FileUploadAmazonS3Server extends FileUploadBase { - constructor(directive, meta, file) { - super(meta, file); - console.log('AmazonS3Server', {directive, meta, file}); - this.store = directive === 'avatar' ? AmazonS3ServerStoreAvatar : AmazonS3ServerStore; - } - - start(callback) { - this.handler = new UploadFS.Uploader({ - store: this.store, - data: this.file, - file: this.meta, - onError: (err) => { - return callback(err); - }, - onComplete: (fileData) => { - const file = _.pick(fileData, '_id', 'type', 'size', 'name', 'identify', 'description'); - - file.url = fileData.url.replace(Meteor.absoluteUrl(), '/'); - return callback(null, file, 'fs'); - } - }); - - this.handler.onProgress = (file, progress) => { - this.onProgress(progress); - }; - - return this.handler.start(); - } - - onProgress() {} - - stop() { - return this.handler.stop(); - } -}; diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorage.js b/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorage.js index a5aa21485b28..9e64169382b9 100644 --- a/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorage.js +++ b/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorage.js @@ -1,42 +1,56 @@ -/* globals FileUpload, FileUploadBase, Slingshot */ +/* globals FileUpload, FileUploadBase, UploadFS, GoogleCloudStorageServerStore:true, GoogleCloudStorageServerStoreAvatar:true */ -FileUpload.GoogleCloudStorage = class FileUploadGoogleCloudStorage extends FileUploadBase { +import '../../ufs/GoogleStorage/client.js'; + +GoogleCloudStorageServerStore = new UploadFS.store.GoogleStorage({ + collection: RocketChat.models.Uploads.model, + name: 'GoogleCloudStorageServer:Uploads', + filter: new UploadFS.Filter({ + onCheck: FileUpload.validateFileUpload + }) +}); + +GoogleCloudStorageServerStoreAvatar = new UploadFS.store.GoogleStorage({ + collection: RocketChat.models.Avatars.model, + name: 'GoogleCloudStorageServer:Avatars', + filter: new UploadFS.Filter({ + onCheck: FileUpload.validateFileUpload + }) +}); + +FileUpload.GoogleCloudStorageServer = class FileUploadGoogleCloudStorageServer extends FileUploadBase { constructor(directive, meta, file) { super(meta, file); - const directives = { - 'upload': 'rocketchat-uploads-gs', - 'avatar': 'rocketchat-avatars-gs' - }; - this.uploader = new Slingshot.Upload(directives[directive], { rid: meta.rid }); + console.log('GoogleCloudStorageServer', {directive, meta, file}); + this.store = directive === 'avatar' ? GoogleCloudStorageServerStoreAvatar : GoogleCloudStorageServerStore; } start(callback) { - this.uploader.send(this.file, (error, downloadUrl) => { - if (this.computation) { - this.computation.stop(); - } + this.handler = new UploadFS.Uploader({ + store: this.store, + data: this.file, + file: this.meta, + onError: (err) => { + return callback(err); + }, + onComplete: (fileData) => { + const file = _.pick(fileData, '_id', 'type', 'size', 'name', 'identify', 'description'); - if (error) { - return callback.call(this, error); - } else { - const file = _.pick(this.meta, 'type', 'size', 'name', 'identify', 'description'); - file._id = downloadUrl.substr(downloadUrl.lastIndexOf('/') + 1); - file.url = downloadUrl; - - return callback(null, file, 'GoogleCloudStorage:Uploads'); + file.url = fileData.url.replace(Meteor.absoluteUrl(), '/'); + return callback(null, file, 'fs'); } }); - this.computation = Tracker.autorun(() => { - this.onProgress(this.uploader.progress()); - }); + this.handler.onProgress = (file, progress) => { + this.onProgress(progress); + }; + + return this.handler.start(); } onProgress() {} stop() { - if (this.uploader && this.uploader.xhr) { - this.uploader.xhr.abort(); - } + return this.handler.stop(); } }; diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorageServer.js b/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorageServer.js deleted file mode 100644 index 9e64169382b9..000000000000 --- a/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorageServer.js +++ /dev/null @@ -1,56 +0,0 @@ -/* globals FileUpload, FileUploadBase, UploadFS, GoogleCloudStorageServerStore:true, GoogleCloudStorageServerStoreAvatar:true */ - -import '../../ufs/GoogleStorage/client.js'; - -GoogleCloudStorageServerStore = new UploadFS.store.GoogleStorage({ - collection: RocketChat.models.Uploads.model, - name: 'GoogleCloudStorageServer:Uploads', - filter: new UploadFS.Filter({ - onCheck: FileUpload.validateFileUpload - }) -}); - -GoogleCloudStorageServerStoreAvatar = new UploadFS.store.GoogleStorage({ - collection: RocketChat.models.Avatars.model, - name: 'GoogleCloudStorageServer:Avatars', - filter: new UploadFS.Filter({ - onCheck: FileUpload.validateFileUpload - }) -}); - -FileUpload.GoogleCloudStorageServer = class FileUploadGoogleCloudStorageServer extends FileUploadBase { - constructor(directive, meta, file) { - super(meta, file); - console.log('GoogleCloudStorageServer', {directive, meta, file}); - this.store = directive === 'avatar' ? GoogleCloudStorageServerStoreAvatar : GoogleCloudStorageServerStore; - } - - start(callback) { - this.handler = new UploadFS.Uploader({ - store: this.store, - data: this.file, - file: this.meta, - onError: (err) => { - return callback(err); - }, - onComplete: (fileData) => { - const file = _.pick(fileData, '_id', 'type', 'size', 'name', 'identify', 'description'); - - file.url = fileData.url.replace(Meteor.absoluteUrl(), '/'); - return callback(null, file, 'fs'); - } - }); - - this.handler.onProgress = (file, progress) => { - this.onProgress(progress); - }; - - return this.handler.start(); - } - - onProgress() {} - - stop() { - return this.handler.stop(); - } -}; diff --git a/packages/rocketchat-file-upload/package.js b/packages/rocketchat-file-upload/package.js index a847f22070d5..e433a70915a8 100644 --- a/packages/rocketchat-file-upload/package.js +++ b/packages/rocketchat-file-upload/package.js @@ -28,10 +28,8 @@ Package.onUse(function(api) { api.addFiles('lib/FileUploadBase.js'); api.addFiles('client/lib/FileUploadAmazonS3.js', 'client'); - api.addFiles('client/lib/FileUploadAmazonS3Server.js', 'client'); api.addFiles('client/lib/FileUploadFileSystem.js', 'client'); api.addFiles('client/lib/FileUploadGoogleStorage.js', 'client'); - api.addFiles('client/lib/FileUploadGoogleStorageServer.js', 'client'); api.addFiles('client/lib/FileUploadGridFS.js', 'client'); api.addFiles('client/lib/fileUploadHandler.js', 'client'); @@ -39,10 +37,10 @@ Package.onUse(function(api) { api.addFiles('server/lib/requests.js', 'server'); api.addFiles('server/config/configFileUploadAmazonS3.js', 'server'); - api.addFiles('server/config/configFileUploadAmazonS3Server.js', 'server'); + api.addFiles('server/config/configFileUploadAmazonS3_Deprecated.js', 'server'); api.addFiles('server/config/configFileUploadFileSystem.js', 'server'); api.addFiles('server/config/configFileUploadGoogleStorage.js', 'server'); - api.addFiles('server/config/configFileUploadGoogleStorageServer.js', 'server'); + api.addFiles('server/config/configFileUploadGoogleStorage_Deprecated.js', 'server'); api.addFiles('server/config/configFileUploadGridFS.js', 'server'); api.addFiles('server/methods/sendFileMessage.js', 'server'); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js index 594f7c5574c7..53f8760c851e 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js @@ -1,36 +1,19 @@ -/* globals Slingshot, FileUpload, AWS */ -import { FileUploadClass } from '../lib/FileUpload'; -import AWS4 from '../lib/AWS4.js'; - -let S3accessKey; -let S3secretKey; -let S3expiryTimeSpan; - -const generateURL = function(file) { - if (!file || !file.s3) { - return; - } +/* globals FileUpload, UploadFS, RocketChatFile */ - const credential = { - accessKeyId: S3accessKey, - secretKey: S3secretKey - }; +import fs from 'fs'; +import { FileUploadClass } from '../lib/FileUpload'; +import '../../ufs/AmazonS3/server.js'; - const req = { - bucket: file.s3.bucket, - region: file.s3.region, - path: `/${ file.s3.path }${ file._id }`, - url: file.url, - expire: Math.max(5, S3expiryTimeSpan) - }; +const Future = Npm.require('fibers/future'); - const queryString = AWS4.sign(req, credential); +const insert = function(file, stream, cb) { + const fileId = this.store.create(file); - return `${ file.url }?${ queryString }`; + this.store.write(stream, fileId, cb); }; -const getFile = function(file, req, res) { - const fileUrl = generateURL(file); +const get = function(file, req, res) { + const fileUrl = this.store.getS3URL(file); if (fileUrl) { res.setHeader('Location', fileUrl); @@ -39,165 +22,142 @@ const getFile = function(file, req, res) { res.end(); }; -const deleteFile = function(file) { - const s3 = new AWS.S3(); - const request = s3.deleteObject({ - Bucket: file.s3.bucket, - Key: file.s3.path + file._id - }); - request.send(); -}; - -// DEPRECATED: backwards compatibility (remove) -new FileUploadClass({ - name: 's3', - model: 'Uploads', - - get: getFile, - delete: deleteFile -}); - -new FileUploadClass({ +const AmazonS3Uploads = new FileUploadClass({ name: 'AmazonS3:Uploads', + // store setted bellow - get: getFile, - delete: deleteFile + get, + insert }); -new FileUploadClass({ +const AmazonS3Avatars = new FileUploadClass({ name: 'AmazonS3:Avatars', + // store setted bellow - get: getFile, - delete: deleteFile + get, + insert }); -function createDirective(directiveName, { key, bucket, accessKey, secretKey, region, acl, cdn, bucketUrl}) { - if (Slingshot._directives[directiveName]) { - delete Slingshot._directives[directiveName]; +const onValidate = function(file) { + if (RocketChatFile.enabled === false || !/^image\/((x-windows-)?bmp|p?jpeg|png)$/.test(file.type)) { + return; } - const config = { - bucket, - key, - AWSAccessKeyId: accessKey, - AWSSecretAccessKey: secretKey - }; - if (!_.isEmpty(acl)) { - config.acl = acl; - } + const tmpFile = UploadFS.getTempFilePath(file._id); - if (!_.isEmpty(cdn)) { - config.cdn = cdn; - } + const fut = new Future(); - if (!_.isEmpty(region)) { - config.region = region; - } + const identify = Meteor.bindEnvironment((err, data) => { + if (err != null) { + console.error(err); + return fut.return(); + } - if (!_.isEmpty(bucketUrl)) { - config.bucketUrl = bucketUrl; - } + file.identify = { + format: data.format, + size: data.size + }; - try { - Slingshot.createDirective(directiveName, Slingshot.S3Storage, config); - } catch (e) { - console.error('Error configuring S3 ->', e.message); - } -} - -const configureSlingshot = _.debounce(() => { - const directives = [ - { - name: 'rocketchat-uploads', - key(file, metaContext) { - const path = `${ RocketChat.hostname }/${ metaContext.rid }/${ this.userId }/`; - - const upload = { - rid: metaContext.rid, - s3: { - bucket: RocketChat.settings.get('FileUpload_S3_Bucket'), - region: RocketChat.settings.get('FileUpload_S3_Region'), - path - } - }; - const fileId = RocketChat.models.Uploads.insertFileInit(this.userId, 'AmazonS3:Uploads', file, upload); - - return path + fileId; - } - }, - { - name: 'rocketchat-avatars', - key(file/*, metaContext*/) { - const path = `${ RocketChat.hostname }/avatars/`; - - const user = RocketChat.models.Users.findOneById(this.userId); - - const upload = { - username: user && user.username, - s3: { - bucket: RocketChat.settings.get('FileUpload_S3_Bucket'), - region: RocketChat.settings.get('FileUpload_S3_Region'), - path - } - }; - delete file.name; - RocketChat.models.Avatars.insertAvatarFileInit(user.username, this.userId, 'AmazonS3:Avatars', file, upload); - - return path + user.username; - } + if ([null, undefined, '', 'Unknown', 'Undefined'].includes(data.Orientation)) { + return fut.return(); } - ]; - - const type = RocketChat.settings.get('FileUpload_Storage_Type'); - const bucket = RocketChat.settings.get('FileUpload_S3_Bucket'); - const acl = RocketChat.settings.get('FileUpload_S3_Acl'); - const accessKey = RocketChat.settings.get('FileUpload_S3_AWSAccessKeyId'); - const secretKey = RocketChat.settings.get('FileUpload_S3_AWSSecretAccessKey'); - const cdn = RocketChat.settings.get('FileUpload_S3_CDN'); - const region = RocketChat.settings.get('FileUpload_S3_Region'); - const bucketUrl = RocketChat.settings.get('FileUpload_S3_BucketURL'); - - AWS.config.update({ - accessKeyId: RocketChat.settings.get('FileUpload_S3_AWSAccessKeyId'), - secretAccessKey: RocketChat.settings.get('FileUpload_S3_AWSSecretAccessKey') - }); - if (type === 'AmazonS3' && !_.isEmpty(bucket) && !_.isEmpty(accessKey) && !_.isEmpty(secretKey)) { - directives.forEach((conf) => { - createDirective(conf.name, { key: conf.key, bucket, accessKey, secretKey, region, acl, cdn, bucketUrl}); - }); - } else { - directives.forEach((conf) => { - if (Slingshot._directives[conf.name]) { - delete Slingshot._directives[conf.name]; + RocketChatFile.gm(tmpFile).autoOrient().write(tmpFile, Meteor.bindEnvironment((err) => { + if (err != null) { + console.error(err); } - }); - } -}, 500); -RocketChat.settings.get('FileUpload_Storage_Type', configureSlingshot); + const size = fs.lstatSync(tmpFile).size; + this.getCollection().direct.update({_id: file._id}, {$set: {size}}); + fut.return(); + })); + }); -RocketChat.settings.get('FileUpload_S3_Bucket', configureSlingshot); + RocketChatFile.gm(tmpFile).identify(identify); -RocketChat.settings.get('FileUpload_S3_Acl', configureSlingshot); + return fut.wait(); +}; -RocketChat.settings.get('FileUpload_S3_AWSAccessKeyId', function(key, value) { - S3accessKey = value; - configureSlingshot(); -}); +const configure = _.debounce(function() { + const stores = UploadFS.getStores(); + delete stores[AmazonS3Uploads.name]; + delete stores[AmazonS3Avatars.name]; -RocketChat.settings.get('FileUpload_S3_AWSSecretAccessKey', function(key, value) { - S3secretKey = value; - configureSlingshot(); -}); + const Bucket = RocketChat.settings.get('FileUpload_S3_Bucket'); + const Acl = RocketChat.settings.get('FileUpload_S3_Acl'); + const AWSAccessKeyId = RocketChat.settings.get('FileUpload_S3_AWSAccessKeyId'); + const AWSSecretAccessKey = RocketChat.settings.get('FileUpload_S3_AWSSecretAccessKey'); + const URLExpiryTimeSpan = RocketChat.settings.get('FileUpload_S3_URLExpiryTimeSpan'); + const Region = RocketChat.settings.get('FileUpload_S3_Region'); + // const CDN = RocketChat.settings.get('FileUpload_S3_CDN'); + // const BucketURL = RocketChat.settings.get('FileUpload_S3_BucketURL'); -RocketChat.settings.get('FileUpload_S3_URLExpiryTimeSpan', function(key, value) { - S3expiryTimeSpan = value; - configureSlingshot(); -}); + const config = { + connection: { + accessKeyId: AWSAccessKeyId, + secretAccessKey: AWSSecretAccessKey, + signatureVersion: 'v4', + params: { + Bucket, + ACL: Acl + }, + region: Region + }, + URLExpiryTimeSpan + }; + + AmazonS3Uploads.store = new UploadFS.store.AmazonS3(Object.assign({ + collection: AmazonS3Uploads.model.model, + filter: new UploadFS.Filter({ + onCheck: FileUpload.validateFileUpload + }), + name: AmazonS3Uploads.name, + onValidate + }, config)); + + AmazonS3Avatars.store = new UploadFS.store.AmazonS3(Object.assign({ + collection: AmazonS3Avatars.model.model, + name: AmazonS3Avatars.name, + onFinishUpload(file) { + // update file record to match user's username + const user = RocketChat.models.Users.findOneById(file.userId); + const oldAvatar = AmazonS3Avatars.model.findOneByName(user.username); + if (oldAvatar) { + try { + AmazonS3Avatars.deleteById(oldAvatar._id); + } catch (e) { + console.error(e); + } + } + AmazonS3Avatars.model.updateFileNameById(file._id, user.username); + // console.log('upload finished ->', file); + }, + onValidate(file) { + if (RocketChatFile.enabled === false || RocketChat.settings.get('Accounts_AvatarResize') !== true) { + return; + } + + const tmpFile = UploadFS.getTempFilePath(file._id); -RocketChat.settings.get('FileUpload_S3_CDN', configureSlingshot); + const fut = new Future(); -RocketChat.settings.get('FileUpload_S3_Region', configureSlingshot); + const height = RocketChat.settings.get('Accounts_AvatarSize'); + const width = height; + + RocketChatFile.gm(tmpFile).background('#ffffff').resize(width, `${ height }^`).gravity('Center').crop(width, height).extent(width, height).setFormat('jpeg').write(tmpFile, Meteor.bindEnvironment((err) => { + if (err != null) { + console.error(err); + } + + const size = fs.lstatSync(tmpFile).size; + this.getCollection().direct.update({_id: file._id}, {$set: {size}}); + fut.return(); + })); + + return fut.wait(); + } + }, config)); + +}, 500); -RocketChat.settings.get('FileUpload_S3_BucketURL', configureSlingshot); +RocketChat.settings.get(/^FileUpload_S3_/, configure); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3Server.js b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3Server.js deleted file mode 100644 index 35bf3ae37392..000000000000 --- a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3Server.js +++ /dev/null @@ -1,163 +0,0 @@ -/* globals FileUpload, UploadFS, RocketChatFile */ - -import fs from 'fs'; -import { FileUploadClass } from '../lib/FileUpload'; -import '../../ufs/AmazonS3/server.js'; - -const Future = Npm.require('fibers/future'); - -const insert = function(file, stream, cb) { - const fileId = this.store.create(file); - - this.store.write(stream, fileId, cb); -}; - -const get = function(file, req, res) { - const fileUrl = this.store.getS3URL(file); - - if (fileUrl) { - res.setHeader('Location', fileUrl); - res.writeHead(302); - } - res.end(); -}; - -const AmazonS3ServerUploads = new FileUploadClass({ - name: 'AmazonS3Server:Uploads', - // store setted bellow - - get, - insert -}); - -const AmazonS3ServerAvatars = new FileUploadClass({ - name: 'AmazonS3Server:Avatars', - // store setted bellow - - get, - insert -}); - -const onValidate = function(file) { - if (RocketChatFile.enabled === false || !/^image\/((x-windows-)?bmp|p?jpeg|png)$/.test(file.type)) { - return; - } - - const tmpFile = UploadFS.getTempFilePath(file._id); - - const fut = new Future(); - - const identify = Meteor.bindEnvironment((err, data) => { - if (err != null) { - console.error(err); - return fut.return(); - } - - file.identify = { - format: data.format, - size: data.size - }; - - if ([null, undefined, '', 'Unknown', 'Undefined'].includes(data.Orientation)) { - return fut.return(); - } - - RocketChatFile.gm(tmpFile).autoOrient().write(tmpFile, Meteor.bindEnvironment((err) => { - if (err != null) { - console.error(err); - } - - const size = fs.lstatSync(tmpFile).size; - this.getCollection().direct.update({_id: file._id}, {$set: {size}}); - fut.return(); - })); - }); - - RocketChatFile.gm(tmpFile).identify(identify); - - return fut.wait(); -}; - -const configure = _.debounce(function() { - const stores = UploadFS.getStores(); - delete stores[AmazonS3ServerUploads.name]; - delete stores[AmazonS3ServerAvatars.name]; - - const Bucket = RocketChat.settings.get('FileUpload_S3_Bucket'); - const Acl = RocketChat.settings.get('FileUpload_S3_Acl'); - const AWSAccessKeyId = RocketChat.settings.get('FileUpload_S3_AWSAccessKeyId'); - const AWSSecretAccessKey = RocketChat.settings.get('FileUpload_S3_AWSSecretAccessKey'); - const URLExpiryTimeSpan = RocketChat.settings.get('FileUpload_S3_URLExpiryTimeSpan'); - const Region = RocketChat.settings.get('FileUpload_S3_Region'); - // const CDN = RocketChat.settings.get('FileUpload_S3_CDN'); - // const BucketURL = RocketChat.settings.get('FileUpload_S3_BucketURL'); - - const config = { - connection: { - accessKeyId: AWSAccessKeyId, - secretAccessKey: AWSSecretAccessKey, - signatureVersion: 'v4', - params: { - Bucket, - ACL: Acl - }, - region: Region - }, - URLExpiryTimeSpan - }; - - AmazonS3ServerUploads.store = new UploadFS.store.AmazonS3(Object.assign({ - collection: AmazonS3ServerUploads.model.model, - filter: new UploadFS.Filter({ - onCheck: FileUpload.validateFileUpload - }), - name: AmazonS3ServerUploads.name, - onValidate - }, config)); - - AmazonS3ServerAvatars.store = new UploadFS.store.AmazonS3(Object.assign({ - collection: AmazonS3ServerAvatars.model.model, - name: AmazonS3ServerAvatars.name, - onFinishUpload(file) { - // update file record to match user's username - const user = RocketChat.models.Users.findOneById(file.userId); - const oldAvatar = AmazonS3ServerAvatars.model.findOneByName(user.username); - if (oldAvatar) { - try { - AmazonS3ServerAvatars.deleteById(oldAvatar._id); - } catch (e) { - console.error(e); - } - } - AmazonS3ServerAvatars.model.updateFileNameById(file._id, user.username); - // console.log('upload finished ->', file); - }, - onValidate(file) { - if (RocketChatFile.enabled === false || RocketChat.settings.get('Accounts_AvatarResize') !== true) { - return; - } - - const tmpFile = UploadFS.getTempFilePath(file._id); - - const fut = new Future(); - - const height = RocketChat.settings.get('Accounts_AvatarSize'); - const width = height; - - RocketChatFile.gm(tmpFile).background('#ffffff').resize(width, `${ height }^`).gravity('Center').crop(width, height).extent(width, height).setFormat('jpeg').write(tmpFile, Meteor.bindEnvironment((err) => { - if (err != null) { - console.error(err); - } - - const size = fs.lstatSync(tmpFile).size; - this.getCollection().direct.update({_id: file._id}, {$set: {size}}); - fut.return(); - })); - - return fut.wait(); - } - }, config)); - -}, 500); - -RocketChat.settings.get(/^FileUpload_S3_/, configure); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3_Deprecated.js b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3_Deprecated.js new file mode 100644 index 000000000000..839151e2fa93 --- /dev/null +++ b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3_Deprecated.js @@ -0,0 +1,189 @@ +/* globals Slingshot, FileUpload, AWS */ +import { FileUploadClass } from '../lib/FileUpload'; +import AWS4 from '../lib/AWS4.js'; + +let S3accessKey; +let S3secretKey; +let S3expiryTimeSpan; + +const generateURL = function(file) { + if (!file || !file.s3) { + return; + } + + const credential = { + accessKeyId: S3accessKey, + secretKey: S3secretKey + }; + + const req = { + bucket: file.s3.bucket, + region: file.s3.region, + path: `/${ file.s3.path }${ file._id }`, + url: file.url, + expire: Math.max(5, S3expiryTimeSpan) + }; + + const queryString = AWS4.sign(req, credential); + + return `${ file.url }?${ queryString }`; +}; + +const getFile = function(file, req, res) { + const fileUrl = generateURL(file); + + if (fileUrl) { + res.setHeader('Location', fileUrl); + res.writeHead(302); + } + res.end(); +}; + +const deleteFile = function(file) { + const s3 = new AWS.S3(); + const request = s3.deleteObject({ + Bucket: file.s3.bucket, + Key: file.s3.path + file._id + }); + request.send(); +}; + +// DEPRECATED: backwards compatibility (remove) +new FileUploadClass({ + name: 's3', + model: 'Uploads', + + get: getFile, + delete: deleteFile +}); + +function createDirective(directiveName, { key, bucket, accessKey, secretKey, region, acl, cdn, bucketUrl}) { + if (Slingshot._directives[directiveName]) { + delete Slingshot._directives[directiveName]; + } + const config = { + bucket, + key, + AWSAccessKeyId: accessKey, + AWSSecretAccessKey: secretKey + }; + + if (!_.isEmpty(acl)) { + config.acl = acl; + } + + if (!_.isEmpty(cdn)) { + config.cdn = cdn; + } + + if (!_.isEmpty(region)) { + config.region = region; + } + + if (!_.isEmpty(bucketUrl)) { + config.bucketUrl = bucketUrl; + } + + try { + Slingshot.createDirective(directiveName, Slingshot.S3Storage, config); + } catch (e) { + console.error('Error configuring S3 ->', e.message); + } +} + +const configureSlingshot = _.debounce(() => { + const directives = [ + { + name: 'rocketchat-uploads', + key(file, metaContext) { + const path = `${ RocketChat.hostname }/${ metaContext.rid }/${ this.userId }/`; + + const upload = { + rid: metaContext.rid, + s3: { + bucket: RocketChat.settings.get('FileUpload_S3_Bucket'), + region: RocketChat.settings.get('FileUpload_S3_Region'), + path + } + }; + const fileId = RocketChat.models.Uploads.insertFileInit(this.userId, 'AmazonS3:Uploads', file, upload); + + return path + fileId; + } + }, + { + name: 'rocketchat-avatars', + key(file/*, metaContext*/) { + const path = `${ RocketChat.hostname }/avatars/`; + + const user = RocketChat.models.Users.findOneById(this.userId); + + const upload = { + username: user && user.username, + s3: { + bucket: RocketChat.settings.get('FileUpload_S3_Bucket'), + region: RocketChat.settings.get('FileUpload_S3_Region'), + path + } + }; + delete file.name; + RocketChat.models.Avatars.insertAvatarFileInit(user.username, this.userId, 'AmazonS3:Avatars', file, upload); + + return path + user.username; + } + } + ]; + + const type = RocketChat.settings.get('FileUpload_Storage_Type'); + const bucket = RocketChat.settings.get('FileUpload_S3_Bucket'); + const acl = RocketChat.settings.get('FileUpload_S3_Acl'); + const accessKey = RocketChat.settings.get('FileUpload_S3_AWSAccessKeyId'); + const secretKey = RocketChat.settings.get('FileUpload_S3_AWSSecretAccessKey'); + const cdn = RocketChat.settings.get('FileUpload_S3_CDN'); + const region = RocketChat.settings.get('FileUpload_S3_Region'); + const bucketUrl = RocketChat.settings.get('FileUpload_S3_BucketURL'); + + AWS.config.update({ + accessKeyId: RocketChat.settings.get('FileUpload_S3_AWSAccessKeyId'), + secretAccessKey: RocketChat.settings.get('FileUpload_S3_AWSSecretAccessKey') + }); + + if (type === 'AmazonS3' && !_.isEmpty(bucket) && !_.isEmpty(accessKey) && !_.isEmpty(secretKey)) { + directives.forEach((conf) => { + createDirective(conf.name, { key: conf.key, bucket, accessKey, secretKey, region, acl, cdn, bucketUrl}); + }); + } else { + directives.forEach((conf) => { + if (Slingshot._directives[conf.name]) { + delete Slingshot._directives[conf.name]; + } + }); + } +}, 500); + +RocketChat.settings.get('FileUpload_Storage_Type', configureSlingshot); + +RocketChat.settings.get('FileUpload_S3_Bucket', configureSlingshot); + +RocketChat.settings.get('FileUpload_S3_Acl', configureSlingshot); + +RocketChat.settings.get('FileUpload_S3_AWSAccessKeyId', function(key, value) { + S3accessKey = value; + configureSlingshot(); +}); + +RocketChat.settings.get('FileUpload_S3_AWSSecretAccessKey', function(key, value) { + S3secretKey = value; + configureSlingshot(); +}); + +RocketChat.settings.get('FileUpload_S3_URLExpiryTimeSpan', function(key, value) { + S3expiryTimeSpan = value; + configureSlingshot(); +}); + +RocketChat.settings.get('FileUpload_S3_CDN', configureSlingshot); + +RocketChat.settings.get('FileUpload_S3_Region', configureSlingshot); + +RocketChat.settings.get('FileUpload_S3_BucketURL', configureSlingshot); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js index af41098cdc7a..7e48b3381835 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js @@ -1,179 +1,161 @@ -/* globals FileUpload, Slingshot */ +/* globals FileUpload, UploadFS, RocketChatFile */ -import crypto from 'crypto'; +import fs from 'fs'; import { FileUploadClass } from '../lib/FileUpload'; +import '../../ufs/GoogleStorage/server.js'; -function generateUrlParts({ file }) { - const accessId = RocketChat.settings.get('FileUpload_GoogleStorage_AccessId'); - const secret = RocketChat.settings.get('FileUpload_GoogleStorage_Secret'); +const Future = Npm.require('fibers/future'); - if (!file || !file.googleCloudStorage || _.isEmpty(accessId) || _.isEmpty(secret)) { - return; - } +const insert = function(file, stream, cb) { + const fileId = this.store.create(file); - return { - accessId: encodeURIComponent(accessId), - secret, - path: file.googleCloudStorage.path + file._id - }; -} + this.store.write(stream, fileId, cb); +}; -function generateGetURL({ file }) { - const parts = generateUrlParts({ file }); +const get = function(file, req, res) { + this.store.getRedirectURL(file, (err, fileUrl) => { + if (err) { + console.error(err); + } - if (!parts) { - return; - } + if (fileUrl) { + res.setHeader('Location', fileUrl); + res.writeHead(302); + } + res.end(); + }); +}; - const expires = new Date().getTime() + 120000; - const signature = crypto.createSign('RSA-SHA256').update(`GET\n\n\n${ expires }\n/${ file.googleCloudStorage.bucket }/${ parts.path }`).sign(parts.secret, 'base64'); +const GoogleCloudStorageUploads = new FileUploadClass({ + name: 'GoogleCloudStorage:Uploads', + // store setted bellow - return `${ file.url }?GoogleAccessId=${ parts.accessId }&Expires=${ expires }&Signature=${ encodeURIComponent(signature) }`; -} + get, + insert +}); -function generateDeleteUrl({ file }) { - const parts = generateUrlParts({ file }); +const GoogleCloudStorageAvatars = new FileUploadClass({ + name: 'GoogleCloudStorage:Avatars', + // store setted bellow - if (!parts) { + get, + insert +}); + +const onValidate = function(file) { + if (RocketChatFile.enabled === false || !/^image\/((x-windows-)?bmp|p?jpeg|png)$/.test(file.type)) { return; } - const expires = new Date().getTime() + 5000; - const signature = crypto.createSign('RSA-SHA256').update(`DELETE\n\n\n${ expires }\n/${ file.googleCloudStorage.bucket }/${ encodeURIComponent(parts.path) }`).sign(parts.secret, 'base64'); + const tmpFile = UploadFS.getTempFilePath(file._id); - return `https://${ file.googleCloudStorage.bucket }.storage.googleapis.com/${ encodeURIComponent(parts.path) }?GoogleAccessId=${ parts.accessId }&Expires=${ expires }&Signature=${ encodeURIComponent(signature) }`; -} + const fut = new Future(); -function createDirective(directiveName, { key, bucket, accessId, secret }) { - if (Slingshot._directives[directiveName]) { - delete Slingshot._directives[directiveName]; - } + const identify = Meteor.bindEnvironment((err, data) => { + if (err != null) { + console.error(err); + return fut.return(); + } - const config = { - bucket, - GoogleAccessId: accessId, - GoogleSecretKey: secret, - key - }; + file.identify = { + format: data.format, + size: data.size + }; - try { - Slingshot.createDirective(directiveName, Slingshot.GoogleCloud, config); - } catch (e) { - console.error('Error configuring GoogleCloudStorage ->', e.message); - } -} + if ([null, undefined, '', 'Unknown', 'Undefined'].includes(data.Orientation)) { + return fut.return(); + } -const getFile = function(file, req, res) { - const fileUrl = generateGetURL({ file }); + RocketChatFile.gm(tmpFile).autoOrient().write(tmpFile, Meteor.bindEnvironment((err) => { + if (err != null) { + console.error(err); + } - if (fileUrl) { - res.setHeader('Location', fileUrl); - res.writeHead(302); - } - res.end(); -}; + const size = fs.lstatSync(tmpFile).size; + this.getCollection().direct.update({_id: file._id}, {$set: {size}}); + fut.return(); + })); + }); -const deleteFile = function(file) { - if (!file || !file.googleCloudStorage) { - console.warn('Failed to delete a file which is uploaded to Google Cloud Storage, the file and googleCloudStorage properties are not defined.'); - return; - } + RocketChatFile.gm(tmpFile).identify(identify); - // RocketChat.models.Uploads.deleteFile(file._id); + return fut.wait(); +}; - const url = generateDeleteUrl({ file }); +const configure = _.debounce(function() { + const stores = UploadFS.getStores(); + delete stores[GoogleCloudStorageUploads.name]; + delete stores[GoogleCloudStorageAvatars.name]; - if (_.isEmpty(url)) { - console.warn('Failed to delete a file which is uploaded to Google Cloud Storage, failed to generate a delete url.'); - return; - } + // const type = RocketChat.settings.get('FileUpload_Storage_Type'); + const bucket = RocketChat.settings.get('FileUpload_GoogleStorage_Bucket'); + const accessId = RocketChat.settings.get('FileUpload_GoogleStorage_AccessId'); + const secret = RocketChat.settings.get('FileUpload_GoogleStorage_Secret'); + const URLExpiryTimeSpan = RocketChat.settings.get('FileUpload_S3_URLExpiryTimeSpan'); - HTTP.call('DELETE', url); -}; + const config = { + connection: { + credentials: { + client_email: accessId, + private_key: secret + } + }, + bucket, + URLExpiryTimeSpan + }; -// DEPRECATED: backwards compatibility (remove) -new FileUploadClass({ - name: 'googleCloudStorage', - model: 'Uploads', + GoogleCloudStorageUploads.store = new UploadFS.store.GoogleStorage(Object.assign({ + collection: GoogleCloudStorageUploads.model.model, + filter: new UploadFS.Filter({ + onCheck: FileUpload.validateFileUpload + }), + name: GoogleCloudStorageUploads.name, + onValidate + }, config)); + + GoogleCloudStorageAvatars.store = new UploadFS.store.GoogleStorage(Object.assign({ + collection: GoogleCloudStorageAvatars.model.model, + name: GoogleCloudStorageAvatars.name, + onFinishUpload(file) { + // update file record to match user's username + const user = RocketChat.models.Users.findOneById(file.userId); + const oldAvatar = GoogleCloudStorageAvatars.model.findOneByName(user.username); + if (oldAvatar) { + try { + GoogleCloudStorageAvatars.deleteById(oldAvatar._id); + } catch (e) { + console.error(e); + } + } + GoogleCloudStorageAvatars.model.updateFileNameById(file._id, user.username); + // console.log('upload finished ->', file); + }, + onValidate(file) { + if (RocketChatFile.enabled === false || RocketChat.settings.get('Accounts_AvatarResize') !== true) { + return; + } - get: getFile, - delete: deleteFile -}); + const tmpFile = UploadFS.getTempFilePath(file._id); -new FileUploadClass({ - name: 'GoogleCloudStorage:Uploads', + const fut = new Future(); - get: getFile, - delete: deleteFile -}); + const height = RocketChat.settings.get('Accounts_AvatarSize'); + const width = height; -new FileUploadClass({ - name: 'GoogleCloudStorage:Avatars', + RocketChatFile.gm(tmpFile).background('#ffffff').resize(width, `${ height }^`).gravity('Center').crop(width, height).extent(width, height).setFormat('jpeg').write(tmpFile, Meteor.bindEnvironment((err) => { + if (err != null) { + console.error(err); + } - get: getFile, - delete: deleteFile -}); + const size = fs.lstatSync(tmpFile).size; + this.getCollection().direct.update({_id: file._id}, {$set: {size}}); + fut.return(); + })); -const createGoogleStorageDirective = _.debounce(() => { - const directives = [ - { - name: 'rocketchat-uploads-gs', - key: function _googleCloudStorageKey(file, metaContext) { - const path = `${ RocketChat.settings.get('uniqueID') }/${ metaContext.rid }/${ this.userId }/`; - const upload = { - rid: metaContext.rid, - googleCloudStorage: { - bucket: RocketChat.settings.get('FileUpload_GoogleStorage_Bucket'), - path - } - }; - const fileId = RocketChat.models.Uploads.insertFileInit(this.userId, 'GoogleCloudStorage:Uploads', file, upload); - - return path + fileId; - } - }, - { - name: 'rocketchat-avatars-gs', - key(file/*, metaContext*/) { - const path = `${ RocketChat.settings.get('uniqueID') }/avatars/`; - - const user = RocketChat.models.Users.findOneById(this.userId); - - const upload = { - username: user && user.username, - googleCloudStorage: { - bucket: RocketChat.settings.get('FileUpload_GoogleStorage_Bucket'), - path - } - }; - delete file.name; - RocketChat.models.Avatars.insertAvatarFileInit(user.username, this.userId, 'GoogleCloudStorage:Avatars', file, upload); - - return path + user.username; - } + return fut.wait(); } - ]; + }, config)); - const type = RocketChat.settings.get('FileUpload_Storage_Type'); - const bucket = RocketChat.settings.get('FileUpload_GoogleStorage_Bucket'); - const accessId = RocketChat.settings.get('FileUpload_GoogleStorage_AccessId'); - const secret = RocketChat.settings.get('FileUpload_GoogleStorage_Secret'); - - if (type === 'GoogleCloudStorage' && !_.isEmpty(secret) && !_.isEmpty(accessId) && !_.isEmpty(bucket)) { - directives.forEach((conf) => { - console.log('conf.name ->', conf.name); - createDirective(conf.name, { key: conf.key, bucket, accessId, secret }); - }); - } else { - directives.forEach((conf) => { - if (Slingshot._directives[conf.name]) { - delete Slingshot._directives[conf.name]; - } - }); - } }, 500); -RocketChat.settings.get('FileUpload_Storage_Type', createGoogleStorageDirective); -RocketChat.settings.get('FileUpload_GoogleStorage_Bucket', createGoogleStorageDirective); -RocketChat.settings.get('FileUpload_GoogleStorage_AccessId', createGoogleStorageDirective); -RocketChat.settings.get('FileUpload_GoogleStorage_Secret', createGoogleStorageDirective); +RocketChat.settings.get(/^FileUpload_GoogleStorage_/, configure); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorageServer.js b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorageServer.js deleted file mode 100644 index d51fb4bbbe36..000000000000 --- a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorageServer.js +++ /dev/null @@ -1,161 +0,0 @@ -/* globals FileUpload, UploadFS, RocketChatFile */ - -import fs from 'fs'; -import { FileUploadClass } from '../lib/FileUpload'; -import '../../ufs/GoogleStorage/server.js'; - -const Future = Npm.require('fibers/future'); - -const insert = function(file, stream, cb) { - const fileId = this.store.create(file); - - this.store.write(stream, fileId, cb); -}; - -const get = function(file, req, res) { - this.store.getRedirectURL(file, (err, fileUrl) => { - if (err) { - console.error(err); - } - - if (fileUrl) { - res.setHeader('Location', fileUrl); - res.writeHead(302); - } - res.end(); - }); -}; - -const GoogleCloudStorageServerUploads = new FileUploadClass({ - name: 'GoogleCloudStorageServer:Uploads', - // store setted bellow - - get, - insert -}); - -const GoogleCloudStorageServerAvatars = new FileUploadClass({ - name: 'GoogleCloudStorageServer:Avatars', - // store setted bellow - - get, - insert -}); - -const onValidate = function(file) { - if (RocketChatFile.enabled === false || !/^image\/((x-windows-)?bmp|p?jpeg|png)$/.test(file.type)) { - return; - } - - const tmpFile = UploadFS.getTempFilePath(file._id); - - const fut = new Future(); - - const identify = Meteor.bindEnvironment((err, data) => { - if (err != null) { - console.error(err); - return fut.return(); - } - - file.identify = { - format: data.format, - size: data.size - }; - - if ([null, undefined, '', 'Unknown', 'Undefined'].includes(data.Orientation)) { - return fut.return(); - } - - RocketChatFile.gm(tmpFile).autoOrient().write(tmpFile, Meteor.bindEnvironment((err) => { - if (err != null) { - console.error(err); - } - - const size = fs.lstatSync(tmpFile).size; - this.getCollection().direct.update({_id: file._id}, {$set: {size}}); - fut.return(); - })); - }); - - RocketChatFile.gm(tmpFile).identify(identify); - - return fut.wait(); -}; - -const configure = _.debounce(function() { - const stores = UploadFS.getStores(); - delete stores[GoogleCloudStorageServerUploads.name]; - delete stores[GoogleCloudStorageServerAvatars.name]; - - // const type = RocketChat.settings.get('FileUpload_Storage_Type'); - const bucket = RocketChat.settings.get('FileUpload_GoogleStorage_Bucket'); - const accessId = RocketChat.settings.get('FileUpload_GoogleStorage_AccessId'); - const secret = RocketChat.settings.get('FileUpload_GoogleStorage_Secret'); - const URLExpiryTimeSpan = RocketChat.settings.get('FileUpload_S3_URLExpiryTimeSpan'); - - const config = { - connection: { - credentials: { - client_email: accessId, - private_key: secret - } - }, - bucket, - URLExpiryTimeSpan - }; - - GoogleCloudStorageServerUploads.store = new UploadFS.store.GoogleStorage(Object.assign({ - collection: GoogleCloudStorageServerUploads.model.model, - filter: new UploadFS.Filter({ - onCheck: FileUpload.validateFileUpload - }), - name: GoogleCloudStorageServerUploads.name, - onValidate - }, config)); - - GoogleCloudStorageServerAvatars.store = new UploadFS.store.GoogleStorage(Object.assign({ - collection: GoogleCloudStorageServerAvatars.model.model, - name: GoogleCloudStorageServerAvatars.name, - onFinishUpload(file) { - // update file record to match user's username - const user = RocketChat.models.Users.findOneById(file.userId); - const oldAvatar = GoogleCloudStorageServerAvatars.model.findOneByName(user.username); - if (oldAvatar) { - try { - GoogleCloudStorageServerAvatars.deleteById(oldAvatar._id); - } catch (e) { - console.error(e); - } - } - GoogleCloudStorageServerAvatars.model.updateFileNameById(file._id, user.username); - // console.log('upload finished ->', file); - }, - onValidate(file) { - if (RocketChatFile.enabled === false || RocketChat.settings.get('Accounts_AvatarResize') !== true) { - return; - } - - const tmpFile = UploadFS.getTempFilePath(file._id); - - const fut = new Future(); - - const height = RocketChat.settings.get('Accounts_AvatarSize'); - const width = height; - - RocketChatFile.gm(tmpFile).background('#ffffff').resize(width, `${ height }^`).gravity('Center').crop(width, height).extent(width, height).setFormat('jpeg').write(tmpFile, Meteor.bindEnvironment((err) => { - if (err != null) { - console.error(err); - } - - const size = fs.lstatSync(tmpFile).size; - this.getCollection().direct.update({_id: file._id}, {$set: {size}}); - fut.return(); - })); - - return fut.wait(); - } - }, config)); - -}, 500); - -RocketChat.settings.get(/^FileUpload_GoogleStorage_/, configure); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage_Deprecated.js b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage_Deprecated.js new file mode 100644 index 000000000000..9c5b9da4b99e --- /dev/null +++ b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage_Deprecated.js @@ -0,0 +1,165 @@ +/* globals FileUpload, Slingshot */ + +import crypto from 'crypto'; +import { FileUploadClass } from '../lib/FileUpload'; + +function generateUrlParts({ file }) { + const accessId = RocketChat.settings.get('FileUpload_GoogleStorage_AccessId'); + const secret = RocketChat.settings.get('FileUpload_GoogleStorage_Secret'); + + if (!file || !file.googleCloudStorage || _.isEmpty(accessId) || _.isEmpty(secret)) { + return; + } + + return { + accessId: encodeURIComponent(accessId), + secret, + path: file.googleCloudStorage.path + file._id + }; +} + +function generateGetURL({ file }) { + const parts = generateUrlParts({ file }); + + if (!parts) { + return; + } + + const expires = new Date().getTime() + 120000; + const signature = crypto.createSign('RSA-SHA256').update(`GET\n\n\n${ expires }\n/${ file.googleCloudStorage.bucket }/${ parts.path }`).sign(parts.secret, 'base64'); + + return `${ file.url }?GoogleAccessId=${ parts.accessId }&Expires=${ expires }&Signature=${ encodeURIComponent(signature) }`; +} + +function generateDeleteUrl({ file }) { + const parts = generateUrlParts({ file }); + + if (!parts) { + return; + } + + const expires = new Date().getTime() + 5000; + const signature = crypto.createSign('RSA-SHA256').update(`DELETE\n\n\n${ expires }\n/${ file.googleCloudStorage.bucket }/${ encodeURIComponent(parts.path) }`).sign(parts.secret, 'base64'); + + return `https://${ file.googleCloudStorage.bucket }.storage.googleapis.com/${ encodeURIComponent(parts.path) }?GoogleAccessId=${ parts.accessId }&Expires=${ expires }&Signature=${ encodeURIComponent(signature) }`; +} + +function createDirective(directiveName, { key, bucket, accessId, secret }) { + if (Slingshot._directives[directiveName]) { + delete Slingshot._directives[directiveName]; + } + + const config = { + bucket, + GoogleAccessId: accessId, + GoogleSecretKey: secret, + key + }; + + try { + Slingshot.createDirective(directiveName, Slingshot.GoogleCloud, config); + } catch (e) { + console.error('Error configuring GoogleCloudStorage ->', e.message); + } +} + +const getFile = function(file, req, res) { + const fileUrl = generateGetURL({ file }); + + if (fileUrl) { + res.setHeader('Location', fileUrl); + res.writeHead(302); + } + res.end(); +}; + +const deleteFile = function(file) { + if (!file || !file.googleCloudStorage) { + console.warn('Failed to delete a file which is uploaded to Google Cloud Storage, the file and googleCloudStorage properties are not defined.'); + return; + } + + // RocketChat.models.Uploads.deleteFile(file._id); + + const url = generateDeleteUrl({ file }); + + if (_.isEmpty(url)) { + console.warn('Failed to delete a file which is uploaded to Google Cloud Storage, failed to generate a delete url.'); + return; + } + + HTTP.call('DELETE', url); +}; + +// DEPRECATED: backwards compatibility (remove) +new FileUploadClass({ + name: 'googleCloudStorage', + model: 'Uploads', + + get: getFile, + delete: deleteFile +}); + +const createGoogleStorageDirective = _.debounce(() => { + const directives = [ + { + name: 'rocketchat-uploads-gs', + key: function _googleCloudStorageKey(file, metaContext) { + const path = `${ RocketChat.settings.get('uniqueID') }/${ metaContext.rid }/${ this.userId }/`; + const upload = { + rid: metaContext.rid, + googleCloudStorage: { + bucket: RocketChat.settings.get('FileUpload_GoogleStorage_Bucket'), + path + } + }; + const fileId = RocketChat.models.Uploads.insertFileInit(this.userId, 'GoogleCloudStorage:Uploads', file, upload); + + return path + fileId; + } + }, + { + name: 'rocketchat-avatars-gs', + key(file/*, metaContext*/) { + const path = `${ RocketChat.settings.get('uniqueID') }/avatars/`; + + const user = RocketChat.models.Users.findOneById(this.userId); + + const upload = { + username: user && user.username, + googleCloudStorage: { + bucket: RocketChat.settings.get('FileUpload_GoogleStorage_Bucket'), + path + } + }; + delete file.name; + RocketChat.models.Avatars.insertAvatarFileInit(user.username, this.userId, 'GoogleCloudStorage:Avatars', file, upload); + + return path + user.username; + } + } + ]; + + const type = RocketChat.settings.get('FileUpload_Storage_Type'); + const bucket = RocketChat.settings.get('FileUpload_GoogleStorage_Bucket'); + const accessId = RocketChat.settings.get('FileUpload_GoogleStorage_AccessId'); + const secret = RocketChat.settings.get('FileUpload_GoogleStorage_Secret'); + + if (type === 'GoogleCloudStorage' && !_.isEmpty(secret) && !_.isEmpty(accessId) && !_.isEmpty(bucket)) { + directives.forEach((conf) => { + console.log('conf.name ->', conf.name); + createDirective(conf.name, { key: conf.key, bucket, accessId, secret }); + }); + } else { + directives.forEach((conf) => { + if (Slingshot._directives[conf.name]) { + delete Slingshot._directives[conf.name]; + } + }); + } +}, 500); + +RocketChat.settings.get('FileUpload_Storage_Type', createGoogleStorageDirective); +RocketChat.settings.get('FileUpload_GoogleStorage_Bucket', createGoogleStorageDirective); +RocketChat.settings.get('FileUpload_GoogleStorage_AccessId', createGoogleStorageDirective); +RocketChat.settings.get('FileUpload_GoogleStorage_Secret', createGoogleStorageDirective); diff --git a/packages/rocketchat-file-upload/server/startup/settings.js b/packages/rocketchat-file-upload/server/startup/settings.js index 1431930a7e7c..fd1cea0b15a6 100644 --- a/packages/rocketchat-file-upload/server/startup/settings.js +++ b/packages/rocketchat-file-upload/server/startup/settings.js @@ -29,15 +29,9 @@ RocketChat.settings.addGroup('FileUpload', function() { }, { key: 'AmazonS3', i18nLabel: 'AmazonS3' - }, { - key: 'AmazonS3Server', - i18nLabel: 'AmazonS3Server' }, { key: 'GoogleCloudStorage', i18nLabel: 'GoogleCloudStorage' - }, { - key: 'GoogleCloudStorageServer', - i18nLabel: 'GoogleCloudStorageServer' }, { key: 'FileSystem', i18nLabel: 'FileSystem' From 5ba6e00b29a10bd26bda0b0e326adc4a875d82aa Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Thu, 25 May 2017 18:07:38 -0300 Subject: [PATCH 19/42] Reduce client code for uploads --- .../client/lib/FileUploadAmazonS3.js | 47 ++----------------- .../client/lib/FileUploadFileSystem.js | 43 ++--------------- .../client/lib/FileUploadGoogleStorage.js | 47 ++----------------- .../client/lib/FileUploadGridFS.js | 35 -------------- .../client/lib/fileUploadHandler.js | 10 ++-- .../lib/FileUploadBase.js | 32 +++++++++++-- .../client/avatar/prompt.js | 2 +- .../rocketchat-ui/client/lib/fileUpload.js | 2 +- 8 files changed, 50 insertions(+), 168 deletions(-) diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js b/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js index ed58adafa345..3c346703ab6e 100644 --- a/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js +++ b/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js @@ -1,56 +1,19 @@ -/* globals FileUpload, FileUploadBase, UploadFS, AmazonS3ServerStore:true, AmazonS3ServerStoreAvatar:true */ +/* globals FileUpload, UploadFS */ import '../../ufs/AmazonS3/client.js'; -AmazonS3ServerStore = new UploadFS.store.AmazonS3({ +new UploadFS.store.AmazonS3({ collection: RocketChat.models.Uploads.model, - name: 'AmazonS3Server:Uploads', + name: 'AmazonS3:Uploads', filter: new UploadFS.Filter({ onCheck: FileUpload.validateFileUpload }) }); -AmazonS3ServerStoreAvatar = new UploadFS.store.AmazonS3({ +new UploadFS.store.AmazonS3({ collection: RocketChat.models.Avatars.model, - name: 'AmazonS3Server:Avatars', + name: 'AmazonS3:Avatars', filter: new UploadFS.Filter({ onCheck: FileUpload.validateFileUpload }) }); - -FileUpload.AmazonS3Server = class FileUploadAmazonS3Server extends FileUploadBase { - constructor(directive, meta, file) { - super(meta, file); - console.log('AmazonS3Server', {directive, meta, file}); - this.store = directive === 'avatar' ? AmazonS3ServerStoreAvatar : AmazonS3ServerStore; - } - - start(callback) { - this.handler = new UploadFS.Uploader({ - store: this.store, - data: this.file, - file: this.meta, - onError: (err) => { - return callback(err); - }, - onComplete: (fileData) => { - const file = _.pick(fileData, '_id', 'type', 'size', 'name', 'identify', 'description'); - - file.url = fileData.url.replace(Meteor.absoluteUrl(), '/'); - return callback(null, file, 'fs'); - } - }); - - this.handler.onProgress = (file, progress) => { - this.onProgress(progress); - }; - - return this.handler.start(); - } - - onProgress() {} - - stop() { - return this.handler.stop(); - } -}; diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadFileSystem.js b/packages/rocketchat-file-upload/client/lib/FileUploadFileSystem.js index 10bb44cf74fb..a4777194f907 100644 --- a/packages/rocketchat-file-upload/client/lib/FileUploadFileSystem.js +++ b/packages/rocketchat-file-upload/client/lib/FileUploadFileSystem.js @@ -1,6 +1,6 @@ -/* globals FileUploadBase, UploadFS, FileUpload:true, FileSystemStore:true, FileSystemStoreAvatar:true */ +/* globals UploadFS, FileUpload:true */ -FileSystemStore = new UploadFS.store.Local({ +new UploadFS.store.Local({ collection: RocketChat.models.Uploads.model, name: 'FileSystem:Uploads', filter: new UploadFS.Filter({ @@ -8,47 +8,10 @@ FileSystemStore = new UploadFS.store.Local({ }) }); -FileSystemStoreAvatar = new UploadFS.store.Local({ +new UploadFS.store.Local({ collection: RocketChat.models.Avatars.model, name: 'FileSystem:Avatars', filter: new UploadFS.Filter({ onCheck: FileUpload.validateFileUpload }) }); - -FileUpload.FileSystem = class FileUploadFileSystem extends FileUploadBase { - constructor(directive, meta, file) { - super(meta, file); - console.log('filesystem', {directive, meta, file}); - this.store = directive === 'avatar' ? FileSystemStoreAvatar : FileSystemStore; - } - - start(callback) { - this.handler = new UploadFS.Uploader({ - store: this.store, - data: this.file, - file: this.meta, - onError: (err) => { - return callback(err); - }, - onComplete: (fileData) => { - const file = _.pick(fileData, '_id', 'type', 'size', 'name', 'identify', 'description'); - - file.url = fileData.url.replace(Meteor.absoluteUrl(), '/'); - return callback(null, file, 'fs'); - } - }); - - this.handler.onProgress = (file, progress) => { - this.onProgress(progress); - }; - - return this.handler.start(); - } - - onProgress() {} - - stop() { - return this.handler.stop(); - } -}; diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorage.js b/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorage.js index 9e64169382b9..f4c398d4eb22 100644 --- a/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorage.js +++ b/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorage.js @@ -1,56 +1,19 @@ -/* globals FileUpload, FileUploadBase, UploadFS, GoogleCloudStorageServerStore:true, GoogleCloudStorageServerStoreAvatar:true */ +/* globals FileUpload, UploadFS */ import '../../ufs/GoogleStorage/client.js'; -GoogleCloudStorageServerStore = new UploadFS.store.GoogleStorage({ +new UploadFS.store.GoogleStorage({ collection: RocketChat.models.Uploads.model, - name: 'GoogleCloudStorageServer:Uploads', + name: 'GoogleCloudStorage:Uploads', filter: new UploadFS.Filter({ onCheck: FileUpload.validateFileUpload }) }); -GoogleCloudStorageServerStoreAvatar = new UploadFS.store.GoogleStorage({ +new UploadFS.store.GoogleStorage({ collection: RocketChat.models.Avatars.model, - name: 'GoogleCloudStorageServer:Avatars', + name: 'GoogleCloudStorage:Avatars', filter: new UploadFS.Filter({ onCheck: FileUpload.validateFileUpload }) }); - -FileUpload.GoogleCloudStorageServer = class FileUploadGoogleCloudStorageServer extends FileUploadBase { - constructor(directive, meta, file) { - super(meta, file); - console.log('GoogleCloudStorageServer', {directive, meta, file}); - this.store = directive === 'avatar' ? GoogleCloudStorageServerStoreAvatar : GoogleCloudStorageServerStore; - } - - start(callback) { - this.handler = new UploadFS.Uploader({ - store: this.store, - data: this.file, - file: this.meta, - onError: (err) => { - return callback(err); - }, - onComplete: (fileData) => { - const file = _.pick(fileData, '_id', 'type', 'size', 'name', 'identify', 'description'); - - file.url = fileData.url.replace(Meteor.absoluteUrl(), '/'); - return callback(null, file, 'fs'); - } - }); - - this.handler.onProgress = (file, progress) => { - this.onProgress(progress); - }; - - return this.handler.start(); - } - - onProgress() {} - - stop() { - return this.handler.stop(); - } -}; diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadGridFS.js b/packages/rocketchat-file-upload/client/lib/FileUploadGridFS.js index 6b8c3755b99e..e69de29bb2d1 100644 --- a/packages/rocketchat-file-upload/client/lib/FileUploadGridFS.js +++ b/packages/rocketchat-file-upload/client/lib/FileUploadGridFS.js @@ -1,35 +0,0 @@ -/* globals FileUploadBase, UploadFS, FileUpload:true */ -FileUpload.GridFS = class FileUploadGridFS extends FileUploadBase { - constructor(directive, meta, file) { - super(meta, file); - this.store = directive === 'avatar' ? Meteor.fileStoreAvatar : Meteor.fileStore; - } - - start(callback) { - this.handler = new UploadFS.Uploader({ - store: this.store, - data: this.file, - file: this.meta, - onError: (err) => { - return callback(err); - }, - onComplete: (fileData) => { - const file = _.pick(fileData, '_id', 'type', 'size', 'name', 'identify', 'description'); - file.url = fileData.url.replace(Meteor.absoluteUrl(), '/'); - return callback(null, file, 'gridfs'); - } - }); - - this.handler.onProgress = (file, progress) => { - this.onProgress(progress); - }; - - return this.handler.start(); - } - - onProgress() {} - - stop() { - return this.handler.stop(); - } -}; diff --git a/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js b/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js index f62e4bb5b93b..babd65d0c2f0 100644 --- a/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js +++ b/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js @@ -1,10 +1,14 @@ -/* globals FileUpload, fileUploadHandler:true */ +/* globals FileUploadBase, UploadFS, fileUploadHandler:true */ /* exported fileUploadHandler */ fileUploadHandler = (directive, meta, file) => { const storageType = RocketChat.settings.get('FileUpload_Storage_Type'); - if (FileUpload[storageType] !== undefined) { - return new FileUpload[storageType](directive, meta, file); + const storeName = `${ storageType }:${ directive }`; + + const store = UploadFS.getStore(storeName); + + if (store) { + return new FileUploadBase(store, meta, file); } }; diff --git a/packages/rocketchat-file-upload/lib/FileUploadBase.js b/packages/rocketchat-file-upload/lib/FileUploadBase.js index 24a304cf2888..29b6d403444c 100644 --- a/packages/rocketchat-file-upload/lib/FileUploadBase.js +++ b/packages/rocketchat-file-upload/lib/FileUploadBase.js @@ -15,10 +15,13 @@ UploadFS.config.defaultStorePermissions = new UploadFS.StorePermissions({ FileUploadBase = class FileUploadBase { - constructor(meta, file) { + constructor(store, meta, file) { this.id = Random.id(); this.meta = meta; this.file = file; + + console.log(store.options.name, {meta, file}); + this.store = store; } getProgress() { @@ -29,11 +32,32 @@ FileUploadBase = class FileUploadBase { return this.meta.name; } - start() { - + start(callback) { + this.handler = new UploadFS.Uploader({ + store: this.store, + data: this.file, + file: this.meta, + onError: (err) => { + return callback(err); + }, + onComplete: (fileData) => { + const file = _.pick(fileData, '_id', 'type', 'size', 'name', 'identify', 'description'); + + file.url = fileData.url.replace(Meteor.absoluteUrl(), '/'); + return callback(null, file, this.store.options.name); + } + }); + + this.handler.onProgress = (file, progress) => { + this.onProgress(progress); + }; + + return this.handler.start(); } - stop() { + onProgress() {} + stop() { + return this.handler.stop(); } }; diff --git a/packages/rocketchat-ui-account/client/avatar/prompt.js b/packages/rocketchat-ui-account/client/avatar/prompt.js index 67d986890774..6875d1d2979a 100644 --- a/packages/rocketchat-ui-account/client/avatar/prompt.js +++ b/packages/rocketchat-ui-account/client/avatar/prompt.js @@ -96,7 +96,7 @@ Template.avatarPrompt.events({ // description: document.getElementById('file-description').value }; - const upload = fileUploadHandler('avatar', record, files[0]); + const upload = fileUploadHandler('Avatars', record, files[0]); // upload.onProgress = (progress) -> // console.log 'progress ->', progress diff --git a/packages/rocketchat-ui/client/lib/fileUpload.js b/packages/rocketchat-ui/client/lib/fileUpload.js index b928171c34be..9c20385e3021 100644 --- a/packages/rocketchat-ui/client/lib/fileUpload.js +++ b/packages/rocketchat-ui/client/lib/fileUpload.js @@ -147,7 +147,7 @@ fileUpload = function(filesToUpload) { description: document.getElementById('file-description').value }; - const upload = fileUploadHandler('upload', record, file.file); + const upload = fileUploadHandler('Uploads', record, file.file); let uploading = Session.get('uploading') || []; uploading.push({ From 2783e70179fee63fbc0b7e39741494159578b83b Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Thu, 25 May 2017 18:33:17 -0300 Subject: [PATCH 20/42] Init GridFS move to upload package --- lib/fileUpload.js | 7 ------- .../client/lib/FileUploadGridFS.js | 18 ++++++++++++++++++ packages/rocketchat-file-upload/package.js | 1 + 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/lib/fileUpload.js b/lib/fileUpload.js index ae75ffa40443..3ba8109a3c23 100644 --- a/lib/fileUpload.js +++ b/lib/fileUpload.js @@ -131,13 +131,6 @@ if (UploadFS) { Meteor.startup(function() { if (Meteor.isServer) { initFileStore(); - } else { - Tracker.autorun(function(c) { - if (Meteor.userId() && RocketChat.settings.cachedCollection.ready.get()) { - initFileStore(); - c.stop(); - } - }); } }); } diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadGridFS.js b/packages/rocketchat-file-upload/client/lib/FileUploadGridFS.js index e69de29bb2d1..8e784207399d 100644 --- a/packages/rocketchat-file-upload/client/lib/FileUploadGridFS.js +++ b/packages/rocketchat-file-upload/client/lib/FileUploadGridFS.js @@ -0,0 +1,18 @@ +/* globals FileUpload, UploadFS */ + +new UploadFS.store.GridFS({ + collection: RocketChat.models.Uploads.model, + name: 'GridFS:Uploads', + collectionName: 'rocketchat_uploads', + filter: new UploadFS.Filter({ + onCheck: FileUpload.validateFileUpload + }) +}); + +new UploadFS.store.GridFS({ + collection: RocketChat.models.Avatars.model, + name: 'GridFS:Avatars', + filter: new UploadFS.Filter({ + onCheck: FileUpload.validateFileUpload + }) +}); diff --git a/packages/rocketchat-file-upload/package.js b/packages/rocketchat-file-upload/package.js index e433a70915a8..09901093207a 100644 --- a/packages/rocketchat-file-upload/package.js +++ b/packages/rocketchat-file-upload/package.js @@ -11,6 +11,7 @@ Package.onUse(function(api) { api.use('ecmascript'); api.use('rocketchat:file'); api.use('jalik:ufs'); + api.use('jalik:ufs-gridfs'); api.use('jalik:ufs-local@0.2.5'); api.use('edgee:slingshot'); api.use('ostrio:cookies'); From 75b0a1969ff2281cf9c701b15fd0697ca31a53b2 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Thu, 25 May 2017 18:53:23 -0300 Subject: [PATCH 21/42] Move GridFS to uploads package and resuse more code --- lib/fileUpload.js | 136 ------------------ .../client/lib/FileUploadGridFS.js | 5 + packages/rocketchat-file-upload/package.js | 1 + .../server/config/configFileUploadAmazonS3.js | 85 +---------- .../config/configFileUploadFileSystem.js | 45 +----- .../config/configFileUploadGoogleStorage.js | 84 +---------- .../server/config/configFileUploadGridFS.js | 61 ++++++++ .../server/lib/FileUpload.js | 106 ++++++++++++++ 8 files changed, 181 insertions(+), 342 deletions(-) delete mode 100644 lib/fileUpload.js diff --git a/lib/fileUpload.js b/lib/fileUpload.js deleted file mode 100644 index 3ba8109a3c23..000000000000 --- a/lib/fileUpload.js +++ /dev/null @@ -1,136 +0,0 @@ -/* globals UploadFS, FileUpload */ -import { Cookies } from 'meteor/ostrio:cookies'; - -if (UploadFS) { - const initFileStore = function() { - const cookie = new Cookies(); - if (Meteor.isClient) { - document.cookie = `rc_uid=${ escape(Meteor.userId()) }; path=/`; - document.cookie = `rc_token=${ escape(Accounts._storedLoginToken()) }; path=/`; - } - - Meteor.fileStore = new UploadFS.store.GridFS({ - collection: RocketChat.models.Uploads.model, - name: 'GridFS:Uploads', - collectionName: 'rocketchat_uploads', - filter: new UploadFS.Filter({ - onCheck: FileUpload.validateFileUpload - }), - transformWrite(readStream, writeStream, fileId, file) { - if (RocketChatFile.enabled === false || !/^image\/.+/.test(file.type)) { - return readStream.pipe(writeStream); - } - - let stream = undefined; - - const identify = function(err, data) { - if (err) { - return stream.pipe(writeStream); - } - - file.identify = { - format: data.format, - size: data.size - }; - - if (data.Orientation && !['', 'Unknown', 'Undefined'].includes(data.Orientation)) { - RocketChatFile.gm(stream).autoOrient().stream().pipe(writeStream); - } else { - stream.pipe(writeStream); - } - }; - - stream = RocketChatFile.gm(readStream).identify(identify).stream(); - }, - - onRead(fileId, file, req, res) { - if (RocketChat.settings.get('FileUpload_ProtectFiles')) { - let uid; - let token; - - if (req && req.headers && req.headers.cookie) { - const rawCookies = req.headers.cookie; - - if (rawCookies) { - uid = cookie.get('rc_uid', rawCookies) ; - token = cookie.get('rc_token', rawCookies); - } - } - - if (!uid) { - uid = req.query.rc_uid; - token = req.query.rc_token; - } - - if (!uid || !token || !RocketChat.models.Users.findOneByIdAndLoginToken(uid, token)) { - res.writeHead(403); - return false; - } - } - - res.setHeader('content-disposition', `attachment; filename="${ encodeURIComponent(file.name) }"`); - return true; - } - }); - - // DEPRECATED: backwards compatibility (remove) - UploadFS.getStores()['rocketchat_uploads'] = UploadFS.getStores()['GridFS:Uploads']; - - Meteor.fileStoreAvatar = new UploadFS.store.GridFS({ - collection: RocketChat.models.Avatars.model, - name: 'GridFS:Avatars', - collectionName: 'rocketchat_avatars', - // filter: new UploadFS.Filter({ - // onCheck: FileUpload.validateFileUpload - // }), - transformWrite: FileUpload.avatarTransformWrite, - - onFinishUpload(file) { - // update file record to match user's username - const user = RocketChat.models.Users.findOneById(file.userId); - const oldAvatar = RocketChat.models.Avatars.findOneByName(user.username); - if (oldAvatar) { - Meteor.fileStoreAvatar.delete(oldAvatar._id); - RocketChat.models.Avatars.deleteFile(oldAvatar._id); - } - RocketChat.models.Avatars.updateFileNameById(file._id, user.username); - // console.log('upload finished ->', file); - }, - - onRead(fileId, file, req, res) { - if (RocketChat.settings.get('FileUpload_ProtectFiles')) { - let uid; - let token; - - if (req && req.headers && req.headers.cookie) { - const rawCookies = req.headers.cookie; - - if (rawCookies) { - uid = cookie.get('rc_uid', rawCookies) ; - token = cookie.get('rc_token', rawCookies); - } - } - - if (!uid) { - uid = req.query.rc_uid; - token = req.query.rc_token; - } - - if (!uid || !token || !RocketChat.models.Users.findOneByIdAndLoginToken(uid, token)) { - res.writeHead(403); - return false; - } - } - - res.setHeader('content-disposition', `attachment; filename="${ encodeURIComponent(file.name) }"`); - return true; - } - }); - }; - - Meteor.startup(function() { - if (Meteor.isServer) { - initFileStore(); - } - }); -} diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadGridFS.js b/packages/rocketchat-file-upload/client/lib/FileUploadGridFS.js index 8e784207399d..a69fef62a044 100644 --- a/packages/rocketchat-file-upload/client/lib/FileUploadGridFS.js +++ b/packages/rocketchat-file-upload/client/lib/FileUploadGridFS.js @@ -16,3 +16,8 @@ new UploadFS.store.GridFS({ onCheck: FileUpload.validateFileUpload }) }); + +Tracker.autorun(function() { + document.cookie = `rc_uid=${ escape(Meteor.userId()) }; path=/`; + document.cookie = `rc_token=${ escape(Accounts._storedLoginToken()) }; path=/`; +}); diff --git a/packages/rocketchat-file-upload/package.js b/packages/rocketchat-file-upload/package.js index 09901093207a..4c4d2df7cda3 100644 --- a/packages/rocketchat-file-upload/package.js +++ b/packages/rocketchat-file-upload/package.js @@ -18,6 +18,7 @@ Package.onUse(function(api) { api.use('peerlibrary:aws-sdk'); api.use('rocketchat:lib'); api.use('random'); + api.use('accounts-base'); api.use('underscore'); api.use('tracker'); api.use('webapp'); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js index 53f8760c851e..2cb8be50254b 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js @@ -1,11 +1,8 @@ /* globals FileUpload, UploadFS, RocketChatFile */ -import fs from 'fs'; import { FileUploadClass } from '../lib/FileUpload'; import '../../ufs/AmazonS3/server.js'; -const Future = Npm.require('fibers/future'); - const insert = function(file, stream, cb) { const fileId = this.store.create(file); @@ -38,46 +35,6 @@ const AmazonS3Avatars = new FileUploadClass({ insert }); -const onValidate = function(file) { - if (RocketChatFile.enabled === false || !/^image\/((x-windows-)?bmp|p?jpeg|png)$/.test(file.type)) { - return; - } - - const tmpFile = UploadFS.getTempFilePath(file._id); - - const fut = new Future(); - - const identify = Meteor.bindEnvironment((err, data) => { - if (err != null) { - console.error(err); - return fut.return(); - } - - file.identify = { - format: data.format, - size: data.size - }; - - if ([null, undefined, '', 'Unknown', 'Undefined'].includes(data.Orientation)) { - return fut.return(); - } - - RocketChatFile.gm(tmpFile).autoOrient().write(tmpFile, Meteor.bindEnvironment((err) => { - if (err != null) { - console.error(err); - } - - const size = fs.lstatSync(tmpFile).size; - this.getCollection().direct.update({_id: file._id}, {$set: {size}}); - fut.return(); - })); - }); - - RocketChatFile.gm(tmpFile).identify(identify); - - return fut.wait(); -}; - const configure = _.debounce(function() { const stores = UploadFS.getStores(); delete stores[AmazonS3Uploads.name]; @@ -112,50 +69,14 @@ const configure = _.debounce(function() { onCheck: FileUpload.validateFileUpload }), name: AmazonS3Uploads.name, - onValidate + onValidate: FileUpload.uploadsOnValidate }, config)); AmazonS3Avatars.store = new UploadFS.store.AmazonS3(Object.assign({ collection: AmazonS3Avatars.model.model, name: AmazonS3Avatars.name, - onFinishUpload(file) { - // update file record to match user's username - const user = RocketChat.models.Users.findOneById(file.userId); - const oldAvatar = AmazonS3Avatars.model.findOneByName(user.username); - if (oldAvatar) { - try { - AmazonS3Avatars.deleteById(oldAvatar._id); - } catch (e) { - console.error(e); - } - } - AmazonS3Avatars.model.updateFileNameById(file._id, user.username); - // console.log('upload finished ->', file); - }, - onValidate(file) { - if (RocketChatFile.enabled === false || RocketChat.settings.get('Accounts_AvatarResize') !== true) { - return; - } - - const tmpFile = UploadFS.getTempFilePath(file._id); - - const fut = new Future(); - - const height = RocketChat.settings.get('Accounts_AvatarSize'); - const width = height; - - RocketChatFile.gm(tmpFile).background('#ffffff').resize(width, `${ height }^`).gravity('Center').crop(width, height).extent(width, height).setFormat('jpeg').write(tmpFile, Meteor.bindEnvironment((err) => { - if (err != null) { - console.error(err); - } - - const size = fs.lstatSync(tmpFile).size; - this.getCollection().direct.update({_id: file._id}, {$set: {size}}); - fut.return(); - })); - - return fut.wait(); - } + onFinishUpload: FileUpload.avatarsOnFinishUpload, + onValidate: FileUpload.avatarsOnValidate }, config)); }, 500); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js b/packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js index 680019a966ba..664f23661b84 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js @@ -78,34 +78,6 @@ const FileSystemAvatars = new FileUploadClass({ }); -const transformWrite = function(readStream, writeStream, fileId, file) { - if (RocketChatFile.enabled === false || !/^image\/((x-windows-)?bmp|p?jpeg|png)$/.test(file.type)) { - return readStream.pipe(writeStream); - } - - let stream = undefined; - - const identify = function(err, data) { - if (err != null) { - return stream.pipe(writeStream); - } - - file.identify = { - format: data.format, - size: data.size - }; - - if ([null, undefined, '', 'Unknown', 'Undefined'].indexOf(data.Orientation) === -1) { - return RocketChatFile.gm(stream).autoOrient().stream().pipe(writeStream); - } else { - return stream.pipe(writeStream); - } - }; - - stream = RocketChatFile.gm(readStream).identify(identify).stream(); - return; -}; - const createFileSystemStore = _.debounce(function() { const stores = UploadFS.getStores(); delete stores['FileSystem:Uploads']; @@ -118,7 +90,7 @@ const createFileSystemStore = _.debounce(function() { onCheck: FileUpload.validateFileUpload }), name: FileSystemUploads.name, - transformWrite + transformWrite: FileUpload.uploadsTransformWrite }); UploadFS.getStores()['fileSystem'] = UploadFS.getStores()[FileSystemUploads.name]; @@ -128,20 +100,7 @@ const createFileSystemStore = _.debounce(function() { collection: FileSystemAvatars.model.model, name: FileSystemAvatars.name, transformWrite: FileUpload.avatarTransformWrite, - onFinishUpload(file) { - // update file record to match user's username - const user = RocketChat.models.Users.findOneById(file.userId); - const oldAvatar = FileSystemAvatars.model.findOneByName(user.username); - if (oldAvatar) { - try { - FileSystemAvatars.deleteById(oldAvatar._id); - } catch (e) { - console.error(e); - } - } - FileSystemAvatars.model.updateFileNameById(file._id, user.username); - // console.log('upload finished ->', file); - } + onFinishUpload: FileUpload.avatarsOnFinishUpload }); }, 500); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js index 7e48b3381835..3e041960c97f 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js @@ -1,10 +1,8 @@ /* globals FileUpload, UploadFS, RocketChatFile */ -import fs from 'fs'; import { FileUploadClass } from '../lib/FileUpload'; import '../../ufs/GoogleStorage/server.js'; -const Future = Npm.require('fibers/future'); const insert = function(file, stream, cb) { const fileId = this.store.create(file); @@ -42,46 +40,6 @@ const GoogleCloudStorageAvatars = new FileUploadClass({ insert }); -const onValidate = function(file) { - if (RocketChatFile.enabled === false || !/^image\/((x-windows-)?bmp|p?jpeg|png)$/.test(file.type)) { - return; - } - - const tmpFile = UploadFS.getTempFilePath(file._id); - - const fut = new Future(); - - const identify = Meteor.bindEnvironment((err, data) => { - if (err != null) { - console.error(err); - return fut.return(); - } - - file.identify = { - format: data.format, - size: data.size - }; - - if ([null, undefined, '', 'Unknown', 'Undefined'].includes(data.Orientation)) { - return fut.return(); - } - - RocketChatFile.gm(tmpFile).autoOrient().write(tmpFile, Meteor.bindEnvironment((err) => { - if (err != null) { - console.error(err); - } - - const size = fs.lstatSync(tmpFile).size; - this.getCollection().direct.update({_id: file._id}, {$set: {size}}); - fut.return(); - })); - }); - - RocketChatFile.gm(tmpFile).identify(identify); - - return fut.wait(); -}; - const configure = _.debounce(function() { const stores = UploadFS.getStores(); delete stores[GoogleCloudStorageUploads.name]; @@ -110,50 +68,14 @@ const configure = _.debounce(function() { onCheck: FileUpload.validateFileUpload }), name: GoogleCloudStorageUploads.name, - onValidate + onValidate: FileUpload.uploadsOnValidate }, config)); GoogleCloudStorageAvatars.store = new UploadFS.store.GoogleStorage(Object.assign({ collection: GoogleCloudStorageAvatars.model.model, name: GoogleCloudStorageAvatars.name, - onFinishUpload(file) { - // update file record to match user's username - const user = RocketChat.models.Users.findOneById(file.userId); - const oldAvatar = GoogleCloudStorageAvatars.model.findOneByName(user.username); - if (oldAvatar) { - try { - GoogleCloudStorageAvatars.deleteById(oldAvatar._id); - } catch (e) { - console.error(e); - } - } - GoogleCloudStorageAvatars.model.updateFileNameById(file._id, user.username); - // console.log('upload finished ->', file); - }, - onValidate(file) { - if (RocketChatFile.enabled === false || RocketChat.settings.get('Accounts_AvatarResize') !== true) { - return; - } - - const tmpFile = UploadFS.getTempFilePath(file._id); - - const fut = new Future(); - - const height = RocketChat.settings.get('Accounts_AvatarSize'); - const width = height; - - RocketChatFile.gm(tmpFile).background('#ffffff').resize(width, `${ height }^`).gravity('Center').crop(width, height).extent(width, height).setFormat('jpeg').write(tmpFile, Meteor.bindEnvironment((err) => { - if (err != null) { - console.error(err); - } - - const size = fs.lstatSync(tmpFile).size; - this.getCollection().direct.update({_id: file._id}, {$set: {size}}); - fut.return(); - })); - - return fut.wait(); - } + onFinishUpload: FileUpload.avatarsOnFinishUpload, + onValidate: FileUpload.avatarsOnValidate }, config)); }, 500); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js b/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js index 49d6b72e61b9..93aac1306438 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js @@ -4,6 +4,9 @@ import zlib from 'zlib'; import util from 'util'; import { FileUploadClass } from '../lib/FileUpload'; +import { Cookies } from 'meteor/ostrio:cookies'; + +const cookie = new Cookies(); const logger = new Logger('FileUpload'); @@ -127,6 +130,64 @@ const readFromGridFS = function(storeName, fileId, file, headers, req, res) { } }; +const onRead = function(fileId, file, req, res) { + if (RocketChat.settings.get('FileUpload_ProtectFiles')) { + let uid; + let token; + + if (req && req.headers && req.headers.cookie) { + const rawCookies = req.headers.cookie; + + if (rawCookies) { + uid = cookie.get('rc_uid', rawCookies) ; + token = cookie.get('rc_token', rawCookies); + } + } + + if (!uid) { + uid = req.query.rc_uid; + token = req.query.rc_token; + } + + if (!uid || !token || !RocketChat.models.Users.findOneByIdAndLoginToken(uid, token)) { + res.writeHead(403); + return false; + } + } + + res.setHeader('content-disposition', `attachment; filename="${ encodeURIComponent(file.name) }"`); + return true; +}; + +Meteor.fileStore = new UploadFS.store.GridFS({ + collection: RocketChat.models.Uploads.model, + name: 'GridFS:Uploads', + collectionName: 'rocketchat_uploads', + filter: new UploadFS.Filter({ + onCheck: FileUpload.validateFileUpload + }), + transformWrite: FileUpload.uploadsTransformWrite, + + onRead +}); + +// DEPRECATED: backwards compatibility (remove) +UploadFS.getStores()['rocketchat_uploads'] = UploadFS.getStores()['GridFS:Uploads']; + +Meteor.fileStoreAvatar = new UploadFS.store.GridFS({ + collection: RocketChat.models.Avatars.model, + name: 'GridFS:Avatars', + collectionName: 'rocketchat_avatars', + // filter: new UploadFS.Filter({ + // onCheck: FileUpload.validateFileUpload + // }), + transformWrite: FileUpload.avatarTransformWrite, + onFinishUpload: FileUpload.avatarsOnFinishUpload, + + onRead +}); + + const insert = function(file, stream, cb) { const fileId = this.store.create(file); diff --git a/packages/rocketchat-file-upload/server/lib/FileUpload.js b/packages/rocketchat-file-upload/server/lib/FileUpload.js index b3e616859c29..256841707567 100644 --- a/packages/rocketchat-file-upload/server/lib/FileUpload.js +++ b/packages/rocketchat-file-upload/server/lib/FileUpload.js @@ -1,6 +1,8 @@ /* globals UploadFS */ +import fs from 'fs'; import mime from 'mime-type/with-db'; +import Future from 'fibers/future'; Object.assign(FileUpload, { handlers: {}, @@ -14,6 +16,110 @@ Object.assign(FileUpload, { return RocketChatFile.gm(readStream).background('#ffffff').resize(width, `${ height }^`).gravity('Center').crop(width, height).extent(width, height).stream('jpeg').pipe(writeStream); }, + avatarsOnValidate(file) { + if (RocketChatFile.enabled === false || RocketChat.settings.get('Accounts_AvatarResize') !== true) { + return; + } + + const tmpFile = UploadFS.getTempFilePath(file._id); + + const fut = new Future(); + + const height = RocketChat.settings.get('Accounts_AvatarSize'); + const width = height; + + RocketChatFile.gm(tmpFile).background('#ffffff').resize(width, `${ height }^`).gravity('Center').crop(width, height).extent(width, height).setFormat('jpeg').write(tmpFile, Meteor.bindEnvironment((err) => { + if (err != null) { + console.error(err); + } + + const size = fs.lstatSync(tmpFile).size; + this.getCollection().direct.update({_id: file._id}, {$set: {size}}); + fut.return(); + })); + + return fut.wait(); + }, + + uploadsTransformWrite(readStream, writeStream, fileId, file) { + if (RocketChatFile.enabled === false || !/^image\/.+/.test(file.type)) { + return readStream.pipe(writeStream); + } + + let stream = undefined; + + const identify = function(err, data) { + if (err) { + return stream.pipe(writeStream); + } + + file.identify = { + format: data.format, + size: data.size + }; + + if (data.Orientation && !['', 'Unknown', 'Undefined'].includes(data.Orientation)) { + RocketChatFile.gm(stream).autoOrient().stream().pipe(writeStream); + } else { + stream.pipe(writeStream); + } + }; + + stream = RocketChatFile.gm(readStream).identify(identify).stream(); + }, + + uploadsOnValidate(file) { + if (RocketChatFile.enabled === false || !/^image\/((x-windows-)?bmp|p?jpeg|png)$/.test(file.type)) { + return; + } + + const tmpFile = UploadFS.getTempFilePath(file._id); + + const fut = new Future(); + + const identify = Meteor.bindEnvironment((err, data) => { + if (err != null) { + console.error(err); + return fut.return(); + } + + file.identify = { + format: data.format, + size: data.size + }; + + if ([null, undefined, '', 'Unknown', 'Undefined'].includes(data.Orientation)) { + return fut.return(); + } + + RocketChatFile.gm(tmpFile).autoOrient().write(tmpFile, Meteor.bindEnvironment((err) => { + if (err != null) { + console.error(err); + } + + const size = fs.lstatSync(tmpFile).size; + this.getCollection().direct.update({_id: file._id}, {$set: {size}}); + fut.return(); + })); + }); + + RocketChatFile.gm(tmpFile).identify(identify); + + return fut.wait(); + }, + + avatarsOnFinishUpload(file) { + // update file record to match user's username + const user = RocketChat.models.Users.findOneById(file.userId); + const oldAvatar = RocketChat.models.Avatars.findOneByName(user.username); + if (oldAvatar) { + this.delete(oldAvatar._id); + RocketChat.models.Avatars.deleteFile(oldAvatar._id); + } + RocketChat.models.Avatars.updateFileNameById(file._id, user.username); + // console.log('upload finished ->', file); + }, + addExtensionTo(file) { if (mime.lookup(file.name) === file.type) { return file; From 1a788f887a26c37f17296de3129f3e1b0a856748 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Thu, 25 May 2017 18:57:38 -0300 Subject: [PATCH 22/42] Uploads: Reuse more code --- .../server/config/configFileUploadAmazonS3.js | 14 ++------------ .../server/config/configFileUploadFileSystem.js | 14 ++------------ .../server/config/configFileUploadGoogleStorage.js | 14 ++------------ .../server/config/configFileUploadGridFS.js | 14 ++------------ .../server/lib/FileUpload.js | 6 ++++++ 5 files changed, 14 insertions(+), 48 deletions(-) diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js index 2cb8be50254b..931551fedbcd 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js @@ -3,12 +3,6 @@ import { FileUploadClass } from '../lib/FileUpload'; import '../../ufs/AmazonS3/server.js'; -const insert = function(file, stream, cb) { - const fileId = this.store.create(file); - - this.store.write(stream, fileId, cb); -}; - const get = function(file, req, res) { const fileUrl = this.store.getS3URL(file); @@ -21,18 +15,14 @@ const get = function(file, req, res) { const AmazonS3Uploads = new FileUploadClass({ name: 'AmazonS3:Uploads', + get // store setted bellow - - get, - insert }); const AmazonS3Avatars = new FileUploadClass({ name: 'AmazonS3:Avatars', + get // store setted bellow - - get, - insert }); const configure = _.debounce(function() { diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js b/packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js index 664f23661b84..5e2d0055829a 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js @@ -3,12 +3,6 @@ import fs from 'fs'; import { FileUploadClass } from '../lib/FileUpload'; -const insert = function(file, stream, cb) { - const fileId = this.store.create(file); - - this.store.write(stream, fileId, cb); -}; - const FileSystemUploads = new FileUploadClass({ name: 'FileSystem:Uploads', // store setted bellow @@ -33,9 +27,7 @@ const FileSystemUploads = new FileUploadClass({ res.end(); return; } - }, - - insert + } }); const FileSystemAvatars = new FileUploadClass({ @@ -72,9 +64,7 @@ const FileSystemAvatars = new FileUploadClass({ res.end(); return; } - }, - - insert + } }); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js index 3e041960c97f..8f7d6cc655e6 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js @@ -4,12 +4,6 @@ import { FileUploadClass } from '../lib/FileUpload'; import '../../ufs/GoogleStorage/server.js'; -const insert = function(file, stream, cb) { - const fileId = this.store.create(file); - - this.store.write(stream, fileId, cb); -}; - const get = function(file, req, res) { this.store.getRedirectURL(file, (err, fileUrl) => { if (err) { @@ -26,18 +20,14 @@ const get = function(file, req, res) { const GoogleCloudStorageUploads = new FileUploadClass({ name: 'GoogleCloudStorage:Uploads', + get // store setted bellow - - get, - insert }); const GoogleCloudStorageAvatars = new FileUploadClass({ name: 'GoogleCloudStorage:Avatars', + get // store setted bellow - - get, - insert }); const configure = _.debounce(function() { diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js b/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js index 93aac1306438..5712d86ffc30 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js @@ -188,12 +188,6 @@ Meteor.fileStoreAvatar = new UploadFS.store.GridFS({ }); -const insert = function(file, stream, cb) { - const fileId = this.store.create(file); - - this.store.write(stream, fileId, cb); -}; - new FileUploadClass({ name: 'GridFS:Uploads', @@ -206,9 +200,7 @@ new FileUploadClass({ 'Content-Length': file.size }; return readFromGridFS(file.store, file._id, file, headers, req, res); - }, - - insert + } }); new FileUploadClass({ @@ -235,7 +227,5 @@ new FileUploadClass({ 'Content-Length': file.size }; return readFromGridFS(file.store, file._id, file, headers, req, res); - }, - - insert + } }); diff --git a/packages/rocketchat-file-upload/server/lib/FileUpload.js b/packages/rocketchat-file-upload/server/lib/FileUpload.js index 256841707567..a03a6b267d26 100644 --- a/packages/rocketchat-file-upload/server/lib/FileUpload.js +++ b/packages/rocketchat-file-upload/server/lib/FileUpload.js @@ -214,4 +214,10 @@ export class FileUploadClass { return this.delete(file._id); } + + insert(file, stream, cb) { + const fileId = this.store.create(file); + + this.store.write(stream, fileId, cb); + } } From 4a86f9786f418ff6f4c8e860d547f3cb12a6e4e4 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Fri, 26 May 2017 11:54:52 -0300 Subject: [PATCH 23/42] Add only options Uploads and Avatars to client --- .../client/lib/FileUploadAmazonS3.js | 19 ------------- .../client/lib/FileUploadFileSystem.js | 17 ----------- .../client/lib/FileUploadGoogleStorage.js | 19 ------------- .../client/lib/FileUploadGridFS.js | 23 --------------- .../client/lib/fileUploadHandler.js | 28 ++++++++++++++++--- packages/rocketchat-file-upload/package.js | 5 +--- .../server/config/configStore.js | 13 +++++++++ 7 files changed, 38 insertions(+), 86 deletions(-) delete mode 100644 packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js delete mode 100644 packages/rocketchat-file-upload/client/lib/FileUploadFileSystem.js delete mode 100644 packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorage.js delete mode 100644 packages/rocketchat-file-upload/client/lib/FileUploadGridFS.js create mode 100644 packages/rocketchat-file-upload/server/config/configStore.js diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js b/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js deleted file mode 100644 index 3c346703ab6e..000000000000 --- a/packages/rocketchat-file-upload/client/lib/FileUploadAmazonS3.js +++ /dev/null @@ -1,19 +0,0 @@ -/* globals FileUpload, UploadFS */ - -import '../../ufs/AmazonS3/client.js'; - -new UploadFS.store.AmazonS3({ - collection: RocketChat.models.Uploads.model, - name: 'AmazonS3:Uploads', - filter: new UploadFS.Filter({ - onCheck: FileUpload.validateFileUpload - }) -}); - -new UploadFS.store.AmazonS3({ - collection: RocketChat.models.Avatars.model, - name: 'AmazonS3:Avatars', - filter: new UploadFS.Filter({ - onCheck: FileUpload.validateFileUpload - }) -}); diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadFileSystem.js b/packages/rocketchat-file-upload/client/lib/FileUploadFileSystem.js deleted file mode 100644 index a4777194f907..000000000000 --- a/packages/rocketchat-file-upload/client/lib/FileUploadFileSystem.js +++ /dev/null @@ -1,17 +0,0 @@ -/* globals UploadFS, FileUpload:true */ - -new UploadFS.store.Local({ - collection: RocketChat.models.Uploads.model, - name: 'FileSystem:Uploads', - filter: new UploadFS.Filter({ - onCheck: FileUpload.validateFileUpload - }) -}); - -new UploadFS.store.Local({ - collection: RocketChat.models.Avatars.model, - name: 'FileSystem:Avatars', - filter: new UploadFS.Filter({ - onCheck: FileUpload.validateFileUpload - }) -}); diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorage.js b/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorage.js deleted file mode 100644 index f4c398d4eb22..000000000000 --- a/packages/rocketchat-file-upload/client/lib/FileUploadGoogleStorage.js +++ /dev/null @@ -1,19 +0,0 @@ -/* globals FileUpload, UploadFS */ - -import '../../ufs/GoogleStorage/client.js'; - -new UploadFS.store.GoogleStorage({ - collection: RocketChat.models.Uploads.model, - name: 'GoogleCloudStorage:Uploads', - filter: new UploadFS.Filter({ - onCheck: FileUpload.validateFileUpload - }) -}); - -new UploadFS.store.GoogleStorage({ - collection: RocketChat.models.Avatars.model, - name: 'GoogleCloudStorage:Avatars', - filter: new UploadFS.Filter({ - onCheck: FileUpload.validateFileUpload - }) -}); diff --git a/packages/rocketchat-file-upload/client/lib/FileUploadGridFS.js b/packages/rocketchat-file-upload/client/lib/FileUploadGridFS.js deleted file mode 100644 index a69fef62a044..000000000000 --- a/packages/rocketchat-file-upload/client/lib/FileUploadGridFS.js +++ /dev/null @@ -1,23 +0,0 @@ -/* globals FileUpload, UploadFS */ - -new UploadFS.store.GridFS({ - collection: RocketChat.models.Uploads.model, - name: 'GridFS:Uploads', - collectionName: 'rocketchat_uploads', - filter: new UploadFS.Filter({ - onCheck: FileUpload.validateFileUpload - }) -}); - -new UploadFS.store.GridFS({ - collection: RocketChat.models.Avatars.model, - name: 'GridFS:Avatars', - filter: new UploadFS.Filter({ - onCheck: FileUpload.validateFileUpload - }) -}); - -Tracker.autorun(function() { - document.cookie = `rc_uid=${ escape(Meteor.userId()) }; path=/`; - document.cookie = `rc_token=${ escape(Accounts._storedLoginToken()) }; path=/`; -}); diff --git a/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js b/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js index babd65d0c2f0..9d3d353e0547 100644 --- a/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js +++ b/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js @@ -1,14 +1,34 @@ /* globals FileUploadBase, UploadFS, fileUploadHandler:true */ /* exported fileUploadHandler */ -fileUploadHandler = (directive, meta, file) => { - const storageType = RocketChat.settings.get('FileUpload_Storage_Type'); +new UploadFS.Store({ + collection: RocketChat.models.Uploads.model, + name: 'Uploads', + filter: new UploadFS.Filter({ + onCheck: FileUpload.validateFileUpload + }) +}); + +new UploadFS.Store({ + collection: RocketChat.models.Avatars.model, + name: 'Avatars', + filter: new UploadFS.Filter({ + onCheck: FileUpload.validateFileUpload + }) +}); - const storeName = `${ storageType }:${ directive }`; - const store = UploadFS.getStore(storeName); +fileUploadHandler = (directive, meta, file) => { + const store = UploadFS.getStore(directive); if (store) { return new FileUploadBase(store, meta, file); + } else { + console.error('Invalid file store', directive); } }; + +Tracker.autorun(function() { + document.cookie = `rc_uid=${ escape(Meteor.userId()) }; path=/`; + document.cookie = `rc_token=${ escape(Accounts._storedLoginToken()) }; path=/`; +}); diff --git a/packages/rocketchat-file-upload/package.js b/packages/rocketchat-file-upload/package.js index 4c4d2df7cda3..20b2b6c6030c 100644 --- a/packages/rocketchat-file-upload/package.js +++ b/packages/rocketchat-file-upload/package.js @@ -29,10 +29,6 @@ Package.onUse(function(api) { api.addFiles('lib/FileUpload.js'); api.addFiles('lib/FileUploadBase.js'); - api.addFiles('client/lib/FileUploadAmazonS3.js', 'client'); - api.addFiles('client/lib/FileUploadFileSystem.js', 'client'); - api.addFiles('client/lib/FileUploadGoogleStorage.js', 'client'); - api.addFiles('client/lib/FileUploadGridFS.js', 'client'); api.addFiles('client/lib/fileUploadHandler.js', 'client'); api.addFiles('server/lib/FileUpload.js', 'server'); @@ -44,6 +40,7 @@ Package.onUse(function(api) { api.addFiles('server/config/configFileUploadGoogleStorage.js', 'server'); api.addFiles('server/config/configFileUploadGoogleStorage_Deprecated.js', 'server'); api.addFiles('server/config/configFileUploadGridFS.js', 'server'); + api.addFiles('server/config/configStore.js', 'server'); api.addFiles('server/methods/sendFileMessage.js', 'server'); api.addFiles('server/methods/getS3FileUrl.js', 'server'); diff --git a/packages/rocketchat-file-upload/server/config/configStore.js b/packages/rocketchat-file-upload/server/config/configStore.js new file mode 100644 index 000000000000..e0c8aaca4646 --- /dev/null +++ b/packages/rocketchat-file-upload/server/config/configStore.js @@ -0,0 +1,13 @@ +/* globals UploadFS */ + +const configStore = _.debounce(() => { + const store = RocketChat.settings.get('FileUpload_Storage_Type'); + + if (store) { + console.log('Setting default file store to', store); + UploadFS.getStores().Avatars = UploadFS.getStore(`${ store }:Avatars`); + UploadFS.getStores().Uploads = UploadFS.getStore(`${ store }:Uploads`); + } +}, 500); + +RocketChat.settings.get(/^FileUpload_/, configStore); From b44a1868394d1cc0918c8734e6cfee0b987018a7 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Fri, 26 May 2017 12:15:58 -0300 Subject: [PATCH 24/42] Uploads: Improve code reusage --- .../server/config/configFileUploadAmazonS3.js | 24 ++------------ .../config/configFileUploadFileSystem.js | 28 ++++------------ .../config/configFileUploadGoogleStorage.js | 25 ++------------ .../server/config/configFileUploadGridFS.js | 19 ++--------- .../server/lib/FileUpload.js | 33 +++++++++++++++++++ 5 files changed, 48 insertions(+), 81 deletions(-) diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js index 931551fedbcd..2a683317dd45 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js @@ -1,4 +1,4 @@ -/* globals FileUpload, UploadFS, RocketChatFile */ +/* globals FileUpload, RocketChatFile */ import { FileUploadClass } from '../lib/FileUpload'; import '../../ufs/AmazonS3/server.js'; @@ -26,10 +26,6 @@ const AmazonS3Avatars = new FileUploadClass({ }); const configure = _.debounce(function() { - const stores = UploadFS.getStores(); - delete stores[AmazonS3Uploads.name]; - delete stores[AmazonS3Avatars.name]; - const Bucket = RocketChat.settings.get('FileUpload_S3_Bucket'); const Acl = RocketChat.settings.get('FileUpload_S3_Acl'); const AWSAccessKeyId = RocketChat.settings.get('FileUpload_S3_AWSAccessKeyId'); @@ -53,22 +49,8 @@ const configure = _.debounce(function() { URLExpiryTimeSpan }; - AmazonS3Uploads.store = new UploadFS.store.AmazonS3(Object.assign({ - collection: AmazonS3Uploads.model.model, - filter: new UploadFS.Filter({ - onCheck: FileUpload.validateFileUpload - }), - name: AmazonS3Uploads.name, - onValidate: FileUpload.uploadsOnValidate - }, config)); - - AmazonS3Avatars.store = new UploadFS.store.AmazonS3(Object.assign({ - collection: AmazonS3Avatars.model.model, - name: AmazonS3Avatars.name, - onFinishUpload: FileUpload.avatarsOnFinishUpload, - onValidate: FileUpload.avatarsOnValidate - }, config)); - + AmazonS3Uploads.store = FileUpload.configureUploadsStore('AmazonS3', AmazonS3Uploads.name, config); + AmazonS3Avatars.store = FileUpload.configureUploadsStore('AmazonS3', AmazonS3Avatars.name, config); }, 500); RocketChat.settings.get(/^FileUpload_S3_/, configure); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js b/packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js index 5e2d0055829a..5469c6382070 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js @@ -69,29 +69,15 @@ const FileSystemAvatars = new FileUploadClass({ const createFileSystemStore = _.debounce(function() { - const stores = UploadFS.getStores(); - delete stores['FileSystem:Uploads']; - delete stores['FileSystem:Avatars']; - - FileSystemUploads.store = new UploadFS.store.Local({ - path: RocketChat.settings.get('FileUpload_FileSystemPath'), //'/tmp/uploads/photos', - collection: FileSystemUploads.model.model, - filter: new UploadFS.Filter({ - onCheck: FileUpload.validateFileUpload - }), - name: FileSystemUploads.name, - transformWrite: FileUpload.uploadsTransformWrite - }); + const options = { + path: RocketChat.settings.get('FileUpload_FileSystemPath') //'/tmp/uploads/photos', + }; - UploadFS.getStores()['fileSystem'] = UploadFS.getStores()[FileSystemUploads.name]; + FileSystemUploads.store = FileUpload.configureUploadsStore('Local', FileSystemUploads.name, options); + FileSystemAvatars.store = FileUpload.configureUploadsStore('Local', FileSystemAvatars.name, options); - FileSystemAvatars.store = new UploadFS.store.Local({ - path: RocketChat.settings.get('FileUpload_FileSystemPath'), //'/tmp/uploads/photos', - collection: FileSystemAvatars.model.model, - name: FileSystemAvatars.name, - transformWrite: FileUpload.avatarTransformWrite, - onFinishUpload: FileUpload.avatarsOnFinishUpload - }); + // DEPRECATED backwards compatibililty (remove) + UploadFS.getStores()['fileSystem'] = UploadFS.getStores()[FileSystemUploads.name]; }, 500); RocketChat.settings.get('FileUpload_FileSystemPath', createFileSystemStore); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js index 8f7d6cc655e6..154ba89e63e0 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js @@ -1,4 +1,4 @@ -/* globals FileUpload, UploadFS, RocketChatFile */ +/* globals FileUpload, RocketChatFile */ import { FileUploadClass } from '../lib/FileUpload'; import '../../ufs/GoogleStorage/server.js'; @@ -31,11 +31,6 @@ const GoogleCloudStorageAvatars = new FileUploadClass({ }); const configure = _.debounce(function() { - const stores = UploadFS.getStores(); - delete stores[GoogleCloudStorageUploads.name]; - delete stores[GoogleCloudStorageAvatars.name]; - - // const type = RocketChat.settings.get('FileUpload_Storage_Type'); const bucket = RocketChat.settings.get('FileUpload_GoogleStorage_Bucket'); const accessId = RocketChat.settings.get('FileUpload_GoogleStorage_AccessId'); const secret = RocketChat.settings.get('FileUpload_GoogleStorage_Secret'); @@ -52,22 +47,8 @@ const configure = _.debounce(function() { URLExpiryTimeSpan }; - GoogleCloudStorageUploads.store = new UploadFS.store.GoogleStorage(Object.assign({ - collection: GoogleCloudStorageUploads.model.model, - filter: new UploadFS.Filter({ - onCheck: FileUpload.validateFileUpload - }), - name: GoogleCloudStorageUploads.name, - onValidate: FileUpload.uploadsOnValidate - }, config)); - - GoogleCloudStorageAvatars.store = new UploadFS.store.GoogleStorage(Object.assign({ - collection: GoogleCloudStorageAvatars.model.model, - name: GoogleCloudStorageAvatars.name, - onFinishUpload: FileUpload.avatarsOnFinishUpload, - onValidate: FileUpload.avatarsOnValidate - }, config)); - + GoogleCloudStorageUploads.store = FileUpload.configureUploadsStore('GoogleStorage', GoogleCloudStorageUploads.name, config); + GoogleCloudStorageAvatars.store = FileUpload.configureUploadsStore('GoogleStorage', GoogleCloudStorageAvatars.name, config); }, 500); RocketChat.settings.get(/^FileUpload_GoogleStorage_/, configure); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js b/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js index 5712d86ffc30..c42d0a29ab60 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js @@ -159,31 +159,16 @@ const onRead = function(fileId, file, req, res) { return true; }; -Meteor.fileStore = new UploadFS.store.GridFS({ - collection: RocketChat.models.Uploads.model, - name: 'GridFS:Uploads', +Meteor.fileStore = FileUpload.configureUploadsStore('GridFS', 'GridFS:Uploads', { collectionName: 'rocketchat_uploads', - filter: new UploadFS.Filter({ - onCheck: FileUpload.validateFileUpload - }), - transformWrite: FileUpload.uploadsTransformWrite, - onRead }); // DEPRECATED: backwards compatibility (remove) UploadFS.getStores()['rocketchat_uploads'] = UploadFS.getStores()['GridFS:Uploads']; -Meteor.fileStoreAvatar = new UploadFS.store.GridFS({ - collection: RocketChat.models.Avatars.model, - name: 'GridFS:Avatars', +Meteor.fileStoreAvatar = FileUpload.configureUploadsStore('GridFS', 'GridFS:Avatars', { collectionName: 'rocketchat_avatars', - // filter: new UploadFS.Filter({ - // onCheck: FileUpload.validateFileUpload - // }), - transformWrite: FileUpload.avatarTransformWrite, - onFinishUpload: FileUpload.avatarsOnFinishUpload, - onRead }); diff --git a/packages/rocketchat-file-upload/server/lib/FileUpload.js b/packages/rocketchat-file-upload/server/lib/FileUpload.js index a03a6b267d26..4686610f1cec 100644 --- a/packages/rocketchat-file-upload/server/lib/FileUpload.js +++ b/packages/rocketchat-file-upload/server/lib/FileUpload.js @@ -7,6 +7,39 @@ import Future from 'fibers/future'; Object.assign(FileUpload, { handlers: {}, + configureUploadsStore(store, name, options) { + const type = name.split(':').pop(); + const stores = UploadFS.getStores(); + delete stores[name]; + + return new UploadFS.store[store](Object.assign({ + name + }, options, FileUpload[`default${ type }`])); + }, + + get defaultUploads() { + return { + collection: RocketChat.models.Uploads.model, + filter: new UploadFS.Filter({ + onCheck: FileUpload.validateFileUpload + }), + // transformWrite: FileUpload.uploadsTransformWrite + onValidate: FileUpload.uploadsOnValidate + }; + }, + + get defaultAvatars() { + return { + collection: RocketChat.models.Avatars.model, + // filter: new UploadFS.Filter({ + // onCheck: FileUpload.validateFileUpload + // }), + // transformWrite: FileUpload.avatarTransformWrite, + onValidate: FileUpload.avatarsOnValidate, + onFinishUpload: FileUpload.avatarsOnFinishUpload + }; + }, + avatarTransformWrite(readStream, writeStream/*, fileId, file*/) { if (RocketChatFile.enabled === false || RocketChat.settings.get('Accounts_AvatarResize') !== true) { return readStream.pipe(writeStream); From e6e7f74c04fc7a1915ec2b36ed6c183ed1af57be Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Sat, 27 May 2017 17:04:59 -0300 Subject: [PATCH 25/42] Uploads: Remove uncessary code --- packages/rocketchat-file-upload/package.js | 1 - .../server/config/configFileUploadAmazonS3.js | 2 +- .../configFileUploadAmazonS3_Deprecated.js | 164 +++------------- ...onfigFileUploadGoogleStorage_Deprecated.js | 149 ++------------ .../rocketchat-file-upload/server/lib/AWS4.js | 175 ----------------- .../server/lib/FileUpload.js | 6 + .../server/methods/getS3FileUrl.js | 34 +--- .../ufs/AmazonS3/server.js | 185 ++++++++++-------- .../ufs/GoogleStorage/server.js | 141 +++++++------ .../.npm/package/npm-shrinkwrap.json | 28 +-- 10 files changed, 257 insertions(+), 628 deletions(-) delete mode 100644 packages/rocketchat-file-upload/server/lib/AWS4.js diff --git a/packages/rocketchat-file-upload/package.js b/packages/rocketchat-file-upload/package.js index 20b2b6c6030c..8a22b09859a4 100644 --- a/packages/rocketchat-file-upload/package.js +++ b/packages/rocketchat-file-upload/package.js @@ -15,7 +15,6 @@ Package.onUse(function(api) { api.use('jalik:ufs-local@0.2.5'); api.use('edgee:slingshot'); api.use('ostrio:cookies'); - api.use('peerlibrary:aws-sdk'); api.use('rocketchat:lib'); api.use('random'); api.use('accounts-base'); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js index 2a683317dd45..b6ba1651dfc4 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js @@ -4,7 +4,7 @@ import { FileUploadClass } from '../lib/FileUpload'; import '../../ufs/AmazonS3/server.js'; const get = function(file, req, res) { - const fileUrl = this.store.getS3URL(file); + const fileUrl = this.store.getRedirectURL(file); if (fileUrl) { res.setHeader('Location', fileUrl); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3_Deprecated.js b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3_Deprecated.js index 839151e2fa93..958bc87cbc57 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3_Deprecated.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3_Deprecated.js @@ -1,61 +1,4 @@ -/* globals Slingshot, FileUpload, AWS */ -import { FileUploadClass } from '../lib/FileUpload'; -import AWS4 from '../lib/AWS4.js'; - -let S3accessKey; -let S3secretKey; -let S3expiryTimeSpan; - -const generateURL = function(file) { - if (!file || !file.s3) { - return; - } - - const credential = { - accessKeyId: S3accessKey, - secretKey: S3secretKey - }; - - const req = { - bucket: file.s3.bucket, - region: file.s3.region, - path: `/${ file.s3.path }${ file._id }`, - url: file.url, - expire: Math.max(5, S3expiryTimeSpan) - }; - - const queryString = AWS4.sign(req, credential); - - return `${ file.url }?${ queryString }`; -}; - -const getFile = function(file, req, res) { - const fileUrl = generateURL(file); - - if (fileUrl) { - res.setHeader('Location', fileUrl); - res.writeHead(302); - } - res.end(); -}; - -const deleteFile = function(file) { - const s3 = new AWS.S3(); - const request = s3.deleteObject({ - Bucket: file.s3.bucket, - Key: file.s3.path + file._id - }); - request.send(); -}; - -// DEPRECATED: backwards compatibility (remove) -new FileUploadClass({ - name: 's3', - model: 'Uploads', - - get: getFile, - delete: deleteFile -}); +/* globals Slingshot, FileUpload */ function createDirective(directiveName, { key, bucket, accessKey, secretKey, region, acl, cdn, bucketUrl}) { if (Slingshot._directives[directiveName]) { @@ -92,48 +35,6 @@ function createDirective(directiveName, { key, bucket, accessKey, secretKey, reg } const configureSlingshot = _.debounce(() => { - const directives = [ - { - name: 'rocketchat-uploads', - key(file, metaContext) { - const path = `${ RocketChat.hostname }/${ metaContext.rid }/${ this.userId }/`; - - const upload = { - rid: metaContext.rid, - s3: { - bucket: RocketChat.settings.get('FileUpload_S3_Bucket'), - region: RocketChat.settings.get('FileUpload_S3_Region'), - path - } - }; - const fileId = RocketChat.models.Uploads.insertFileInit(this.userId, 'AmazonS3:Uploads', file, upload); - - return path + fileId; - } - }, - { - name: 'rocketchat-avatars', - key(file/*, metaContext*/) { - const path = `${ RocketChat.hostname }/avatars/`; - - const user = RocketChat.models.Users.findOneById(this.userId); - - const upload = { - username: user && user.username, - s3: { - bucket: RocketChat.settings.get('FileUpload_S3_Bucket'), - region: RocketChat.settings.get('FileUpload_S3_Region'), - path - } - }; - delete file.name; - RocketChat.models.Avatars.insertAvatarFileInit(user.username, this.userId, 'AmazonS3:Avatars', file, upload); - - return path + user.username; - } - } - ]; - const type = RocketChat.settings.get('FileUpload_Storage_Type'); const bucket = RocketChat.settings.get('FileUpload_S3_Bucket'); const acl = RocketChat.settings.get('FileUpload_S3_Acl'); @@ -143,47 +44,36 @@ const configureSlingshot = _.debounce(() => { const region = RocketChat.settings.get('FileUpload_S3_Region'); const bucketUrl = RocketChat.settings.get('FileUpload_S3_BucketURL'); - AWS.config.update({ - accessKeyId: RocketChat.settings.get('FileUpload_S3_AWSAccessKeyId'), - secretAccessKey: RocketChat.settings.get('FileUpload_S3_AWSSecretAccessKey') - }); + delete Slingshot._directives['rocketchat-uploads']; if (type === 'AmazonS3' && !_.isEmpty(bucket) && !_.isEmpty(accessKey) && !_.isEmpty(secretKey)) { - directives.forEach((conf) => { - createDirective(conf.name, { key: conf.key, bucket, accessKey, secretKey, region, acl, cdn, bucketUrl}); - }); - } else { - directives.forEach((conf) => { - if (Slingshot._directives[conf.name]) { - delete Slingshot._directives[conf.name]; - } + createDirective('rocketchat-uploads', { + key(file, metaContext) { + const id = Random.id(); + const path = `${ RocketChat.settings.get('uniqueID') }/uploads/${ metaContext.rid }/${ this.userId }/${ id }`; + + const upload = { + _id: id, + rid: metaContext.rid, + AmazonS3: { + path + } + }; + + RocketChat.models.Uploads.insertFileInit(this.userId, 'AmazonS3:Uploads', file, upload); + + return path; + }, + bucket, + accessKey, + secretKey, + region, + acl, + cdn, + bucketUrl }); } }, 500); RocketChat.settings.get('FileUpload_Storage_Type', configureSlingshot); - -RocketChat.settings.get('FileUpload_S3_Bucket', configureSlingshot); - -RocketChat.settings.get('FileUpload_S3_Acl', configureSlingshot); - -RocketChat.settings.get('FileUpload_S3_AWSAccessKeyId', function(key, value) { - S3accessKey = value; - configureSlingshot(); -}); - -RocketChat.settings.get('FileUpload_S3_AWSSecretAccessKey', function(key, value) { - S3secretKey = value; - configureSlingshot(); -}); - -RocketChat.settings.get('FileUpload_S3_URLExpiryTimeSpan', function(key, value) { - S3expiryTimeSpan = value; - configureSlingshot(); -}); - -RocketChat.settings.get('FileUpload_S3_CDN', configureSlingshot); - -RocketChat.settings.get('FileUpload_S3_Region', configureSlingshot); - -RocketChat.settings.get('FileUpload_S3_BucketURL', configureSlingshot); +RocketChat.settings.get(/^FileUpload_S3_/, configureSlingshot); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage_Deprecated.js b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage_Deprecated.js index 9c5b9da4b99e..c4fcda5ecf08 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage_Deprecated.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage_Deprecated.js @@ -1,49 +1,5 @@ /* globals FileUpload, Slingshot */ -import crypto from 'crypto'; -import { FileUploadClass } from '../lib/FileUpload'; - -function generateUrlParts({ file }) { - const accessId = RocketChat.settings.get('FileUpload_GoogleStorage_AccessId'); - const secret = RocketChat.settings.get('FileUpload_GoogleStorage_Secret'); - - if (!file || !file.googleCloudStorage || _.isEmpty(accessId) || _.isEmpty(secret)) { - return; - } - - return { - accessId: encodeURIComponent(accessId), - secret, - path: file.googleCloudStorage.path + file._id - }; -} - -function generateGetURL({ file }) { - const parts = generateUrlParts({ file }); - - if (!parts) { - return; - } - - const expires = new Date().getTime() + 120000; - const signature = crypto.createSign('RSA-SHA256').update(`GET\n\n\n${ expires }\n/${ file.googleCloudStorage.bucket }/${ parts.path }`).sign(parts.secret, 'base64'); - - return `${ file.url }?GoogleAccessId=${ parts.accessId }&Expires=${ expires }&Signature=${ encodeURIComponent(signature) }`; -} - -function generateDeleteUrl({ file }) { - const parts = generateUrlParts({ file }); - - if (!parts) { - return; - } - - const expires = new Date().getTime() + 5000; - const signature = crypto.createSign('RSA-SHA256').update(`DELETE\n\n\n${ expires }\n/${ file.googleCloudStorage.bucket }/${ encodeURIComponent(parts.path) }`).sign(parts.secret, 'base64'); - - return `https://${ file.googleCloudStorage.bucket }.storage.googleapis.com/${ encodeURIComponent(parts.path) }?GoogleAccessId=${ parts.accessId }&Expires=${ expires }&Signature=${ encodeURIComponent(signature) }`; -} - function createDirective(directiveName, { key, bucket, accessId, secret }) { if (Slingshot._directives[directiveName]) { delete Slingshot._directives[directiveName]; @@ -63,103 +19,38 @@ function createDirective(directiveName, { key, bucket, accessId, secret }) { } } -const getFile = function(file, req, res) { - const fileUrl = generateGetURL({ file }); - - if (fileUrl) { - res.setHeader('Location', fileUrl); - res.writeHead(302); - } - res.end(); -}; - -const deleteFile = function(file) { - if (!file || !file.googleCloudStorage) { - console.warn('Failed to delete a file which is uploaded to Google Cloud Storage, the file and googleCloudStorage properties are not defined.'); - return; - } - - // RocketChat.models.Uploads.deleteFile(file._id); - - const url = generateDeleteUrl({ file }); - - if (_.isEmpty(url)) { - console.warn('Failed to delete a file which is uploaded to Google Cloud Storage, failed to generate a delete url.'); - return; - } - - HTTP.call('DELETE', url); -}; - -// DEPRECATED: backwards compatibility (remove) -new FileUploadClass({ - name: 'googleCloudStorage', - model: 'Uploads', - - get: getFile, - delete: deleteFile -}); - const createGoogleStorageDirective = _.debounce(() => { - const directives = [ - { - name: 'rocketchat-uploads-gs', - key: function _googleCloudStorageKey(file, metaContext) { - const path = `${ RocketChat.settings.get('uniqueID') }/${ metaContext.rid }/${ this.userId }/`; - const upload = { - rid: metaContext.rid, - googleCloudStorage: { - bucket: RocketChat.settings.get('FileUpload_GoogleStorage_Bucket'), - path - } - }; - const fileId = RocketChat.models.Uploads.insertFileInit(this.userId, 'GoogleCloudStorage:Uploads', file, upload); + const type = RocketChat.settings.get('FileUpload_Storage_Type'); + const bucket = RocketChat.settings.get('FileUpload_GoogleStorage_Bucket'); + const accessId = RocketChat.settings.get('FileUpload_GoogleStorage_AccessId'); + const secret = RocketChat.settings.get('FileUpload_GoogleStorage_Secret'); - return path + fileId; - } - }, - { - name: 'rocketchat-avatars-gs', - key(file/*, metaContext*/) { - const path = `${ RocketChat.settings.get('uniqueID') }/avatars/`; + delete Slingshot._directives['rocketchat-uploads-gs']; - const user = RocketChat.models.Users.findOneById(this.userId); + if (type === 'GoogleCloudStorage' && !_.isEmpty(secret) && !_.isEmpty(accessId) && !_.isEmpty(bucket)) { + createDirective('rocketchat-uploads-gs', { + key: function _googleCloudStorageKey(file, metaContext) { + const id = Random.id(); + const path = `${ RocketChat.settings.get('uniqueID') }/uploads/${ metaContext.rid }/${ this.userId }/${ id }`; const upload = { - username: user && user.username, - googleCloudStorage: { - bucket: RocketChat.settings.get('FileUpload_GoogleStorage_Bucket'), + _id: id, + rid: metaContext.rid, + GoogleStorage: { path } }; - delete file.name; - RocketChat.models.Avatars.insertAvatarFileInit(user.username, this.userId, 'GoogleCloudStorage:Avatars', file, upload); - return path + user.username; - } - } - ]; + RocketChat.models.Uploads.insertFileInit(this.userId, 'GoogleCloudStorage:Uploads', file, upload); - const type = RocketChat.settings.get('FileUpload_Storage_Type'); - const bucket = RocketChat.settings.get('FileUpload_GoogleStorage_Bucket'); - const accessId = RocketChat.settings.get('FileUpload_GoogleStorage_AccessId'); - const secret = RocketChat.settings.get('FileUpload_GoogleStorage_Secret'); - - if (type === 'GoogleCloudStorage' && !_.isEmpty(secret) && !_.isEmpty(accessId) && !_.isEmpty(bucket)) { - directives.forEach((conf) => { - console.log('conf.name ->', conf.name); - createDirective(conf.name, { key: conf.key, bucket, accessId, secret }); - }); - } else { - directives.forEach((conf) => { - if (Slingshot._directives[conf.name]) { - delete Slingshot._directives[conf.name]; - } + return path; + }, + bucket, + accessId, + secret }); } }, 500); RocketChat.settings.get('FileUpload_Storage_Type', createGoogleStorageDirective); -RocketChat.settings.get('FileUpload_GoogleStorage_Bucket', createGoogleStorageDirective); -RocketChat.settings.get('FileUpload_GoogleStorage_AccessId', createGoogleStorageDirective); -RocketChat.settings.get('FileUpload_GoogleStorage_Secret', createGoogleStorageDirective); +RocketChat.settings.get(/^FileUpload_GoogleStorage_/, createGoogleStorageDirective); diff --git a/packages/rocketchat-file-upload/server/lib/AWS4.js b/packages/rocketchat-file-upload/server/lib/AWS4.js deleted file mode 100644 index bb8980ac0d4c..000000000000 --- a/packages/rocketchat-file-upload/server/lib/AWS4.js +++ /dev/null @@ -1,175 +0,0 @@ -import crypto from 'crypto'; -import urllib from 'url'; -import querystring from 'querystring'; - -const Algorithm = 'AWS4-HMAC-SHA256'; -const DefaultRegion = 'us-east-1'; -const Service = 's3'; -const KeyPartsRequest = 'aws4_request'; - -class Aws4 { - constructor(req, credentials) { - const { url, method = 'GET', body = '', date, region, headers = {}, expire = 86400 } = this.req = req; - - Object.assign(this, { url, body, method: method.toUpperCase() }); - - const urlObj = urllib.parse(url); - this.region = region || DefaultRegion; - this.path = urlObj.pathname; - this.host = urlObj.host; - this.date = date || this.amzDate; - this.credentials = credentials; - this.headers = this.prepareHeaders(headers); - this.expire = expire; - } - - prepareHeaders() { - const host = this.host; - - return { - host - }; - } - - hmac(key, string, encoding) { - return crypto.createHmac('sha256', key).update(string, 'utf8').digest(encoding); - } - - hash(string, encoding = 'hex') { - return crypto.createHash('sha256').update(string, 'utf8').digest(encoding); - } - - encodeRfc3986(urlEncodedString) { - return urlEncodedString.replace(/[!'()*]/g, function(c) { - return `%${ c.charCodeAt(0).toString(16).toUpperCase() }`; - }); - } - - encodeQuery(query) { - return this.encodeRfc3986(querystring.stringify(Object.keys(query).sort().reduce((obj, key) => { - if (!key) { return obj; } - obj[key] = !Array.isArray(query[key]) ? query[key] : query[key].slice().sort(); - return obj; - }, {}))); - } - - get query() { - const query = {}; - - if (this.credentials.sessionToken) { - query['X-Amz-Security-Token'] = this.credentials.sessionToken; - } - - query['X-Amz-Expires'] = this.expire; - query['X-Amz-Date'] = this.amzDate; - query['X-Amz-Algorithm'] = Algorithm; - query['X-Amz-Credential'] = `${ this.credentials.accessKeyId }/${ this.credentialScope }`; - query['X-Amz-SignedHeaders'] = this.signedHeaders; - - return query; - } - - get amzDate() { - return (new Date()).toISOString().replace(/[:\-]|\.\d{3}/g, ''); - } - - get dateStamp() { - return this.date.slice(0, 8); - } - - get payloadHash() { - return 'UNSIGNED-PAYLOAD'; - } - - get canonicalPath() { - let pathStr = this.path; - if (pathStr === '/') { return pathStr; } - - pathStr = pathStr.replace(/\/{2,}/g, '/'); - pathStr = pathStr.split('/').reduce((path, piece) => { - if (piece === '..') { - path.pop(); - } else { - path.push(this.encodeRfc3986(querystring.escape(piece))); - } - return path; - }, []).join('/'); - - return pathStr; - } - - get canonicalQuery() { - return this.encodeQuery(this.query); - } - - get canonicalHeaders() { - const headers = Object.keys(this.headers) - .sort((a, b) => a.toLowerCase() < b.toLowerCase() ? -1 : 1) - .map(key => `${ key.toLowerCase() }:${ this.headers[key] }`); - return `${ headers.join('\n') }\n`; - } - - get signedHeaders() { - return Object.keys(this.headers) - .map(key => key.toLowerCase()) - .sort() - .join(';'); - } - - get canonicalRequest() { - return [ - this.method, - this.canonicalPath, - this.canonicalQuery, - this.canonicalHeaders, - this.signedHeaders, - this.payloadHash - ].join('\n'); - } - - get credentialScope() { - return [ - this.dateStamp, - this.region, - Service, - KeyPartsRequest - ].join('/'); - } - - get stringToSign() { - return [ - Algorithm, - this.date, - this.credentialScope, - this.hash(this.canonicalRequest) - ].join('\n'); - } - - get signingKey() { - const kDate = this.hmac(`AWS4${ this.credentials.secretKey }`, this.dateStamp); - const kRegion = this.hmac(kDate, this.region); - const kService = this.hmac(kRegion, Service); - const kSigning = this.hmac(kService, KeyPartsRequest); - - return kSigning; - } - - get signature() { - return this.hmac(this.signingKey, this.stringToSign, 'hex'); - } - - // Export - // Return signed query string - sign() { - const query = this.query; - query['X-Amz-Signature'] = this.signature; - - return this.encodeQuery(query); - } -} - -export default { - sign(request, credential) { - return (new Aws4(request, credential)).sign(); - } -}; diff --git a/packages/rocketchat-file-upload/server/lib/FileUpload.js b/packages/rocketchat-file-upload/server/lib/FileUpload.js index 4686610f1cec..d189ece30be0 100644 --- a/packages/rocketchat-file-upload/server/lib/FileUpload.js +++ b/packages/rocketchat-file-upload/server/lib/FileUpload.js @@ -23,6 +23,9 @@ Object.assign(FileUpload, { filter: new UploadFS.Filter({ onCheck: FileUpload.validateFileUpload }), + getPath(file) { + return `${ RocketChat.settings.get('uniqueID') }/uploads/${ file.rid }/${ file.userId }/${ file._id }`; + }, // transformWrite: FileUpload.uploadsTransformWrite onValidate: FileUpload.uploadsOnValidate }; @@ -35,6 +38,9 @@ Object.assign(FileUpload, { // onCheck: FileUpload.validateFileUpload // }), // transformWrite: FileUpload.avatarTransformWrite, + getPath(file) { + return `${ RocketChat.settings.get('uniqueID') }/avatars/${ file._id }`; + }, onValidate: FileUpload.avatarsOnValidate, onFinishUpload: FileUpload.avatarsOnFinishUpload }; diff --git a/packages/rocketchat-file-upload/server/methods/getS3FileUrl.js b/packages/rocketchat-file-upload/server/methods/getS3FileUrl.js index c3cbca844eb8..421740d18b84 100644 --- a/packages/rocketchat-file-upload/server/methods/getS3FileUrl.js +++ b/packages/rocketchat-file-upload/server/methods/getS3FileUrl.js @@ -1,26 +1,11 @@ -import AWS4 from '../lib/AWS4.js'; +/* globals UploadFS */ let protectedFiles; -let S3accessKey; -let S3secretKey; -let S3expiryTimeSpan; RocketChat.settings.get('FileUpload_ProtectFiles', function(key, value) { protectedFiles = value; }); -RocketChat.settings.get('FileUpload_S3_AWSAccessKeyId', function(key, value) { - S3accessKey = value; -}); - -RocketChat.settings.get('FileUpload_S3_AWSSecretAccessKey', function(key, value) { - S3secretKey = value; -}); - -RocketChat.settings.get('FileUpload_S3_URLExpiryTimeSpan', function(key, value) { - S3expiryTimeSpan = value; -}); - Meteor.methods({ getS3FileUrl(fileId) { if (protectedFiles && !Meteor.userId()) { @@ -28,21 +13,6 @@ Meteor.methods({ } const file = RocketChat.models.Uploads.findOneById(fileId); - const credential = { - accessKeyId: S3accessKey, - secretKey: S3secretKey - }; - - const req = { - bucket: file.s3.bucket, - region: file.s3.region, - path: `/${ file.s3.path }${ file._id }`, - url: file.url, - expire: Math.max(5, S3expiryTimeSpan) - }; - - const queryString = AWS4.sign(req, credential); - - return `${ file.url }?${ queryString }`; + return UploadFS.getStore('AmazonS3:Uploads').getRedirectURL(file); } }); diff --git a/packages/rocketchat-file-upload/ufs/AmazonS3/server.js b/packages/rocketchat-file-upload/ufs/AmazonS3/server.js index 6dbbd4412cd4..ab3332efb519 100644 --- a/packages/rocketchat-file-upload/ufs/AmazonS3/server.js +++ b/packages/rocketchat-file-upload/ufs/AmazonS3/server.js @@ -1,5 +1,4 @@ import {_} from 'meteor/underscore'; -import {Meteor} from 'meteor/meteor'; import {UploadFS} from 'meteor/jalik:ufs'; import S3 from 'aws-sdk/clients/s3'; import stream from 'stream'; @@ -29,97 +28,123 @@ export class AmazonS3Store extends UploadFS.Store { const classOptions = options; - if (Meteor.isServer) { - const s3 = new S3(options.connection); + const s3 = new S3(options.connection); - this.getPath = function(file) { - return `${ RocketChat.hostname }/${ file.rid }/${ file.userId }/${ file._id }`; + options.getPath = options.getPath || function(file) { + return file._id; + }; + + this.getPath = function(file) { + if (file.AmazonS3) { + return file.AmazonS3.path; + } + // Compatibility + // TODO: Migration + if (file.s3) { + return file.s3.path + file._id; + } + }; + + this.getRedirectURL = function(file) { + const params = { + Key: this.getPath(file), + Expires: classOptions.URLExpiryTimeSpan }; - this.getS3URL = function(file) { - const params = { - Key: this.getPath(file), - Expires: classOptions.URLExpiryTimeSpan - }; + return s3.getSignedUrl('getObject', params); + }; - return s3.getSignedUrl('getObject', params); + /** + * Creates the file in the collection + * @param file + * @param callback + * @return {string} + */ + this.create = function(file, callback) { + check(file, Object); + + file.AmazonS3 = { + path: this.options.getPath(file) }; - /** - * Removes the file - * @param fileId - * @param callback - */ - this.delete = function(fileId, callback) { - const file = this.getCollection().findOne({_id: fileId}); - const params = { - Key: this.getPath(file) - }; - - s3.deleteObject(params, (err, data) => { - if (err) { - console.error(err); - } - - callback && callback(err, data); - }); + file.store = this.options.name; // assign store to file + return this.getCollection().insert(file, callback); + }; + + /** + * Removes the file + * @param fileId + * @param callback + */ + this.delete = function(fileId, callback) { + const file = this.getCollection().findOne({_id: fileId}); + const params = { + Key: this.getPath(file) }; - /** - * Returns the file read stream - * @param fileId - * @param file - * @param options - * @return {*} - */ - this.getReadStream = function(fileId, file, options = {}) { - const params = { - Key: this.getPath(file) - }; - - if (options.start && options.end) { - params.Range = `${ options.start } - ${ options.end }`; + s3.deleteObject(params, (err, data) => { + if (err) { + console.error(err); } - return s3.getObject(params).createReadStream(); + callback && callback(err, data); + }); + }; + + /** + * Returns the file read stream + * @param fileId + * @param file + * @param options + * @return {*} + */ + this.getReadStream = function(fileId, file, options = {}) { + const params = { + Key: this.getPath(file) }; - /** - * Returns the file write stream - * @param fileId - * @param file - * @param options - * @return {*} - */ - this.getWriteStream = function(fileId, file/*, options*/) { - const writeStream = new stream.PassThrough(); - writeStream.length = file.size; - - writeStream.on('newListener', (event, listener) => { - if (event === 'finish') { - process.nextTick(() => { - writeStream.removeListener(event, listener); - writeStream.on('real_finish', listener); - }); - } - }); - - s3.putObject({ - Key: this.getPath(file), - Body: writeStream, - ContentType: file.type - - }, (error) => { - if (error) { - console.error(error); - } - - writeStream.emit('real_finish'); - }); - - return writeStream; - }; - } + if (options.start && options.end) { + params.Range = `${ options.start } - ${ options.end }`; + } + + return s3.getObject(params).createReadStream(); + }; + + /** + * Returns the file write stream + * @param fileId + * @param file + * @param options + * @return {*} + */ + this.getWriteStream = function(fileId, file/*, options*/) { + const writeStream = new stream.PassThrough(); + writeStream.length = file.size; + + writeStream.on('newListener', (event, listener) => { + if (event === 'finish') { + process.nextTick(() => { + writeStream.removeListener(event, listener); + writeStream.on('real_finish', listener); + }); + } + }); + + s3.putObject({ + Key: this.getPath(file), + Body: writeStream, + ContentType: file.type + + }, (error) => { + if (error) { + console.error(error); + } + + writeStream.emit('real_finish'); + }); + + return writeStream; + }; } } diff --git a/packages/rocketchat-file-upload/ufs/GoogleStorage/server.js b/packages/rocketchat-file-upload/ufs/GoogleStorage/server.js index f2cef3e11616..484f80a50d58 100644 --- a/packages/rocketchat-file-upload/ufs/GoogleStorage/server.js +++ b/packages/rocketchat-file-upload/ufs/GoogleStorage/server.js @@ -1,4 +1,3 @@ -import {Meteor} from 'meteor/meteor'; import {UploadFS} from 'meteor/jalik:ufs'; import gcStorage from '@google-cloud/storage'; @@ -12,74 +11,98 @@ export class GoogleStorageStore extends UploadFS.Store { constructor(options) { super(options); - const classOptions = options; + const gcs = gcStorage(options.connection); + this.bucket = gcs.bucket(options.bucket); - if (Meteor.isServer) { - const gcs = gcStorage(options.connection); - const bucket = gcs.bucket(options.bucket); + options.getPath = options.getPath || function(file) { + return file._id; + }; - this.getPath = function(file) { - return `${ RocketChat.hostname }/${ file.rid }/${ file.userId }/${ file._id }`; + this.getPath = function(file) { + if (file.GoogleStorage) { + return file.GoogleStorage.path; + } + // Compatibility + // TODO: Migration + if (file.googleCloudStorage) { + return file.googleCloudStorage.path + file._id; + } + }; + + this.getRedirectURL = function(file, callback) { + const params = { + action: 'read', + responseDisposition: 'inline', + expires: Date.now()+this.options.URLExpiryTimeSpan*1000 }; - this.getRedirectURL = function(file, callback) { - const params = { - action: 'read', - responseDisposition: 'inline', - expires: Date.now()+classOptions.URLExpiryTimeSpan*1000 - }; + this.bucket.file(this.getPath(file)).getSignedUrl(params, callback); + }; + + /** + * Creates the file in the collection + * @param file + * @param callback + * @return {string} + */ + this.create = function(file, callback) { + check(file, Object); - bucket.file(this.getPath(file)).getSignedUrl(params, callback); + file.GoogleStorage = { + path: this.options.getPath(file) }; - /** - * Removes the file - * @param fileId - * @param callback - */ - this.delete = function(fileId, callback) { - const file = this.getCollection().findOne({_id: fileId}); - bucket.file(this.getPath(file)).delete(function(err, data) { - if (err) { - console.error(err); - } + file.store = this.options.name; // assign store to file + return this.getCollection().insert(file, callback); + }; - callback && callback(err, data); - }); - }; + /** + * Removes the file + * @param fileId + * @param callback + */ + this.delete = function(fileId, callback) { + const file = this.getCollection().findOne({_id: fileId}); + this.bucket.file(this.getPath(file)).delete(function(err, data) { + if (err) { + console.error(err); + } - /** - * Returns the file read stream - * @param fileId - * @param file - * @param options - * @return {*} - */ - this.getReadStream = function(fileId, file/*, options = {}*/) { - // TODO range? - return bucket.file(this.getPath(file)).createReadStream(); - }; + callback && callback(err, data); + }); + }; - /** - * Returns the file write stream - * @param fileId - * @param file - * @param options - * @return {*} - */ - this.getWriteStream = function(fileId, file/*, options*/) { - return bucket.file(this.getPath(file)).createWriteStream({ - gzip: false, - metadata: { - contentType: file.type, - contentDisposition: `inline; filename=${ file.name }` - // metadata: { - // custom: 'metadata' - // } - } - }); - }; - } + /** + * Returns the file read stream + * @param fileId + * @param file + * @param options + * @return {*} + */ + this.getReadStream = function(fileId, file/*, options = {}*/) { + // TODO range? + return this.bucket.file(this.getPath(file)).createReadStream(); + }; + + /** + * Returns the file write stream + * @param fileId + * @param file + * @param options + * @return {*} + */ + this.getWriteStream = function(fileId, file/*, options*/) { + return this.bucket.file(this.getPath(file)).createWriteStream({ + gzip: false, + metadata: { + contentType: file.type, + contentDisposition: `inline; filename=${ file.name }` + // metadata: { + // custom: 'metadata' + // } + } + }); + }; } } diff --git a/packages/rocketchat-ui-master/.npm/package/npm-shrinkwrap.json b/packages/rocketchat-ui-master/.npm/package/npm-shrinkwrap.json index 7db964c3bb1a..6a2270165462 100644 --- a/packages/rocketchat-ui-master/.npm/package/npm-shrinkwrap.json +++ b/packages/rocketchat-ui-master/.npm/package/npm-shrinkwrap.json @@ -1,28 +1,28 @@ { "dependencies": { "clipboard": { - "version": "1.5.12", - "resolved": "https://registry.npmjs.org/clipboard/-/clipboard-1.5.12.tgz", - "from": "clipboard@1.5.12" + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/clipboard/-/clipboard-1.6.1.tgz", + "from": "clipboard@1.6.1" }, "delegate": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/delegate/-/delegate-3.1.0.tgz", - "from": "delegate@>=3.1.0 <4.0.0" + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/delegate/-/delegate-3.1.2.tgz", + "from": "delegate@>=3.1.2 <4.0.0" }, "good-listener": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/good-listener/-/good-listener-1.2.0.tgz", - "from": "good-listener@>=1.1.6 <2.0.0" + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/good-listener/-/good-listener-1.2.2.tgz", + "from": "good-listener@>=1.2.0 <2.0.0" }, "select": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/select/-/select-1.1.0.tgz", - "from": "select@>=1.0.6 <2.0.0" + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/select/-/select-1.1.2.tgz", + "from": "select@>=1.1.2 <2.0.0" }, "tiny-emitter": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/tiny-emitter/-/tiny-emitter-1.1.0.tgz", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/tiny-emitter/-/tiny-emitter-1.2.0.tgz", "from": "tiny-emitter@>=1.0.0 <2.0.0" } } From e2666af5247b0251a74f7fcd147590020819199c Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Sat, 27 May 2017 17:10:39 -0300 Subject: [PATCH 26/42] Uploads: Use correct store for import and slack bridge --- .../server/config/configFileUploadGridFS.js | 4 ++-- .../rocketchat-importer/server/classes/ImporterBase.coffee | 5 +++-- packages/rocketchat-slackbridge/slackbridge.js | 7 ++++--- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js b/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js index c42d0a29ab60..ce312f11d866 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js @@ -159,7 +159,7 @@ const onRead = function(fileId, file, req, res) { return true; }; -Meteor.fileStore = FileUpload.configureUploadsStore('GridFS', 'GridFS:Uploads', { +FileUpload.configureUploadsStore('GridFS', 'GridFS:Uploads', { collectionName: 'rocketchat_uploads', onRead }); @@ -167,7 +167,7 @@ Meteor.fileStore = FileUpload.configureUploadsStore('GridFS', 'GridFS:Uploads', // DEPRECATED: backwards compatibility (remove) UploadFS.getStores()['rocketchat_uploads'] = UploadFS.getStores()['GridFS:Uploads']; -Meteor.fileStoreAvatar = FileUpload.configureUploadsStore('GridFS', 'GridFS:Avatars', { +FileUpload.configureUploadsStore('GridFS', 'GridFS:Avatars', { collectionName: 'rocketchat_avatars', onRead }); diff --git a/packages/rocketchat-importer/server/classes/ImporterBase.coffee b/packages/rocketchat-importer/server/classes/ImporterBase.coffee index 197e76d48ee8..a7eff1e2752b 100644 --- a/packages/rocketchat-importer/server/classes/ImporterBase.coffee +++ b/packages/rocketchat-importer/server/classes/ImporterBase.coffee @@ -161,9 +161,10 @@ Importer.Base = class Importer.Base requestModule = if /https/i.test(fileUrl) then Importer.Base.https else Importer.Base.http requestModule.get fileUrl, Meteor.bindEnvironment((stream) -> - fileId = Meteor.fileStore.create details + fileStore = UploadFS.getStore('Uploads') + fileId = fileStore.create details if fileId - Meteor.fileStore.write stream, fileId, (err, file) -> + fileStore.write stream, fileId, (err, file) -> if err throw new Error(err) else diff --git a/packages/rocketchat-slackbridge/slackbridge.js b/packages/rocketchat-slackbridge/slackbridge.js index b4bfa1686deb..cacd3a3e9953 100644 --- a/packages/rocketchat-slackbridge/slackbridge.js +++ b/packages/rocketchat-slackbridge/slackbridge.js @@ -1,4 +1,4 @@ -/* globals logger */ +/* globals logger, UploadFS */ class SlackBridge { @@ -592,9 +592,10 @@ class SlackBridge { const parsedUrl = url.parse(slackFileURL, true); parsedUrl.headers = { 'Authorization': `Bearer ${ this.apiToken }` }; requestModule.get(parsedUrl, Meteor.bindEnvironment((stream) => { - const fileId = Meteor.fileStore.create(details); + const fileStore = UploadFS.getStore('Uploads'); + const fileId = fileStore.create(details); if (fileId) { - Meteor.fileStore.write(stream, fileId, (err, file) => { + fileStore.write(stream, fileId, (err, file) => { if (err) { throw new Error(err); } else { From e9ba87d8651a46c524e04bbe207861bf6cfa3037 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Sat, 27 May 2017 19:29:48 -0300 Subject: [PATCH 27/42] Uploads: Migration --- server/startup/migrations/v096.js | 44 +++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 server/startup/migrations/v096.js diff --git a/server/startup/migrations/v096.js b/server/startup/migrations/v096.js new file mode 100644 index 000000000000..a10674a3b9cf --- /dev/null +++ b/server/startup/migrations/v096.js @@ -0,0 +1,44 @@ +RocketChat.Migrations.add({ + version: 96, + up() { + const query = { + $or: [{ + 's3.path': { + $exists: true + } + }, { + 'googleCloudStorage.path': { + $exists: true + } + }] + }; + + RocketChat.models.Uploads.find(query).forEach((record) => { + if (record.s3) { + RocketChat.models.Uploads.model.direct.update({_id: record._id}, { + $set: { + 'store': 'AmazonS3:Uploads', + AmazonS3: { + path: record.s3.path + record._id + } + }, + $unset: { + s3: 1 + } + }, {multi: true}); + } else { + RocketChat.models.Uploads.model.direct.update({_id: record._id}, { + $set: { + store: 'GoogleCloudStorage:Uploads', + GoogleStorage: { + path: record.googleCloudStorage.path + record._id + } + }, + $unset: { + googleCloudStorage: 1 + } + }, {multi: true}); + } + }); + } +}); From 7b5d45ddca83ae2e3dd97f5da270c885496a75a4 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Sun, 28 May 2017 18:58:07 -0300 Subject: [PATCH 28/42] Uploads: Migrate old FS and GridFS store names --- server/startup/migrations/v096.js | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/server/startup/migrations/v096.js b/server/startup/migrations/v096.js index a10674a3b9cf..6016a3899fc9 100644 --- a/server/startup/migrations/v096.js +++ b/server/startup/migrations/v096.js @@ -40,5 +40,24 @@ RocketChat.Migrations.add({ }, {multi: true}); } }); + + RocketChat.models.Uploads.model.direct.update({ + store: 'fileSystem' + }, { + $set: { + store: 'FileSystem:Uploads' + } + }, { + multi: true + }); + RocketChat.models.Uploads.model.direct.update({ + store: 'rocketchat_uploads' + }, { + $set: { + store: 'GridFS:Uploads' + } + }, { + multi: true + }); } }); From 9ee948e41562c6978c50573208592121be87f5f6 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Sun, 28 May 2017 20:30:58 -0300 Subject: [PATCH 29/42] Uploads: Fix avatar upload from services --- .../server/config/configStore.js | 2 +- .../server/lib/FileUpload.js | 14 ++++--- .../client/avatar/prompt.js | 22 ++++++++--- server/methods/setAvatarFromService.js | 39 ------------------- 4 files changed, 25 insertions(+), 52 deletions(-) diff --git a/packages/rocketchat-file-upload/server/config/configStore.js b/packages/rocketchat-file-upload/server/config/configStore.js index e0c8aaca4646..1fc3e1bdbb98 100644 --- a/packages/rocketchat-file-upload/server/config/configStore.js +++ b/packages/rocketchat-file-upload/server/config/configStore.js @@ -8,6 +8,6 @@ const configStore = _.debounce(() => { UploadFS.getStores().Avatars = UploadFS.getStore(`${ store }:Avatars`); UploadFS.getStores().Uploads = UploadFS.getStore(`${ store }:Uploads`); } -}, 500); +}, 1000); RocketChat.settings.get(/^FileUpload_/, configStore); diff --git a/packages/rocketchat-file-upload/server/lib/FileUpload.js b/packages/rocketchat-file-upload/server/lib/FileUpload.js index d189ece30be0..7cd010daa7bd 100644 --- a/packages/rocketchat-file-upload/server/lib/FileUpload.js +++ b/packages/rocketchat-file-upload/server/lib/FileUpload.js @@ -14,10 +14,10 @@ Object.assign(FileUpload, { return new UploadFS.store[store](Object.assign({ name - }, options, FileUpload[`default${ type }`])); + }, options, FileUpload[`default${ type }`]())); }, - get defaultUploads() { + defaultUploads() { return { collection: RocketChat.models.Uploads.model, filter: new UploadFS.Filter({ @@ -31,7 +31,7 @@ Object.assign(FileUpload, { }; }, - get defaultAvatars() { + defaultAvatars() { return { collection: RocketChat.models.Avatars.model, // filter: new UploadFS.Filter({ @@ -39,7 +39,7 @@ Object.assign(FileUpload, { // }), // transformWrite: FileUpload.avatarTransformWrite, getPath(file) { - return `${ RocketChat.settings.get('uniqueID') }/avatars/${ file._id }`; + return `${ RocketChat.settings.get('uniqueID') }/avatars/${ file.userId }`; }, onValidate: FileUpload.avatarsOnValidate, onFinishUpload: FileUpload.avatarsOnFinishUpload @@ -152,7 +152,6 @@ Object.assign(FileUpload, { const user = RocketChat.models.Users.findOneById(file.userId); const oldAvatar = RocketChat.models.Avatars.findOneByName(user.username); if (oldAvatar) { - this.delete(oldAvatar._id); RocketChat.models.Avatars.deleteFile(oldAvatar._id); } RocketChat.models.Avatars.updateFileNameById(file._id, user.username); @@ -201,7 +200,10 @@ export class FileUploadClass { this.model = model || this.getModelFromName(); this._store = store || UploadFS.getStore(name); this.get = get; - this.insert = insert; + + if (insert) { + this.insert = insert; + } if (getStore) { this.getStore = getStore; diff --git a/packages/rocketchat-ui-account/client/avatar/prompt.js b/packages/rocketchat-ui-account/client/avatar/prompt.js index 6875d1d2979a..6fdd1bdcc5f3 100644 --- a/packages/rocketchat-ui-account/client/avatar/prompt.js +++ b/packages/rocketchat-ui-account/client/avatar/prompt.js @@ -51,7 +51,7 @@ Template.avatarPrompt.events({ 'click .select-service'(event, instance) { if (this.service === 'initials') { Meteor.call('resetAvatar', function(err) { - if (err && err.details.timeToReset && err.details.timeToReset) { + if (err && err.details && err.details.timeToReset) { toastr.error(t('error-too-many-requests', { seconds: parseInt(err.details.timeToReset / 1000) })); @@ -79,7 +79,7 @@ Template.avatarPrompt.events({ } else { toastr.error(t('Please_enter_value_for_url')); } - } else { + } else if (this.service === 'upload') { let files = instance.find('input[type=file]').files; if (!files || files.length === 0) { files = event.dataTransfer && event.dataTransfer.files || []; @@ -103,10 +103,20 @@ Template.avatarPrompt.events({ upload.start((error, result) => { if (result) { - Meteor.call('saveAvatarFile', result, () => { - toastr.success(t('Avatar_changed_successfully')); - RocketChat.callbacks.run('userAvatarSet', this.service); - }); + toastr.success(t('Avatar_changed_successfully')); + RocketChat.callbacks.run('userAvatarSet', this.service); + } + }); + } else { + const tmpService = this.service; + Meteor.call('setAvatarFromService', this.blob, this.contentType, this.service, function(err) { + if (err && err.details && err.details.timeToReset) { + toastr.error(t('error-too-many-requests', { + seconds: parseInt(err.details.timeToReset / 1000) + })); + } else { + toastr.success(t('Avatar_changed_successfully')); + RocketChat.callbacks.run('userAvatarSet', tmpService); } }); } diff --git a/server/methods/setAvatarFromService.js b/server/methods/setAvatarFromService.js index 6f974d1a35cc..6dc13b719e2e 100644 --- a/server/methods/setAvatarFromService.js +++ b/server/methods/setAvatarFromService.js @@ -19,45 +19,6 @@ Meteor.methods({ const user = Meteor.user(); return RocketChat.setUserAvatar(user, dataURI, contentType, service); - }, - - saveAvatarFile(file) { - check(file, Match.ObjectIncluding({ - _id: String - })); - - if (!Meteor.userId()) { - throw new Meteor.Error('error-invalid-user', 'Invalid user', { - method: 'setAvatarFromService' - }); - } - - if (!RocketChat.settings.get('Accounts_AllowUserAvatarChange')) { - throw new Meteor.Error('error-not-allowed', 'Not allowed', { - method: 'setAvatarFromService' - }); - } - - const user = RocketChat.models.Users.findOneById(Meteor.userId()); - const fileSaved = RocketChat.models.Avatars.findOneById(file._id); - - if (!fileSaved) { - return; - } - - if (fileSaved.userId !== user._id) { - // this file is not user's avatar - throw new Meteor.Error('invalid-avatar'); - } - - // just returns if file already complete (case for GridFS) - if (fileSaved.complete) { - return true; - } - - RocketChat.models.Avatars.updateFileCompleteByNameAndUserId(user.username, user._id, file.url); - - return true; } }); From 0ce0be0f023bd1e16226443601e299461371d4b0 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Mon, 29 May 2017 10:43:15 -0300 Subject: [PATCH 30/42] Uploads: Fix review --- .../client/lib/fileUploadHandler.js | 6 +- .../lib/FileUploadBase.js | 2 - .../server/lib/FileUpload.js | 31 +++++- .../ufs/AmazonS3/server.js | 4 + .../ufs/GoogleStorage/server.js | 4 + .../server/classes/ImporterBase.coffee | 85 ++++++++--------- .../server/functions/setUserAvatar.js | 7 +- .../rocketchat-slackbridge/slackbridge.js | 94 +++++++++---------- 8 files changed, 131 insertions(+), 102 deletions(-) diff --git a/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js b/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js index 9d3d353e0547..73474ab5181f 100644 --- a/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js +++ b/packages/rocketchat-file-upload/client/lib/fileUploadHandler.js @@ -29,6 +29,8 @@ fileUploadHandler = (directive, meta, file) => { }; Tracker.autorun(function() { - document.cookie = `rc_uid=${ escape(Meteor.userId()) }; path=/`; - document.cookie = `rc_token=${ escape(Accounts._storedLoginToken()) }; path=/`; + if (Meteor.userId()) { + document.cookie = `rc_uid=${ escape(Meteor.userId()) }; path=/`; + document.cookie = `rc_token=${ escape(Accounts._storedLoginToken()) }; path=/`; + } }); diff --git a/packages/rocketchat-file-upload/lib/FileUploadBase.js b/packages/rocketchat-file-upload/lib/FileUploadBase.js index 29b6d403444c..486fe8592d31 100644 --- a/packages/rocketchat-file-upload/lib/FileUploadBase.js +++ b/packages/rocketchat-file-upload/lib/FileUploadBase.js @@ -19,8 +19,6 @@ FileUploadBase = class FileUploadBase { this.id = Random.id(); this.meta = meta; this.file = file; - - console.log(store.options.name, {meta, file}); this.store = store; } diff --git a/packages/rocketchat-file-upload/server/lib/FileUpload.js b/packages/rocketchat-file-upload/server/lib/FileUpload.js index 7cd010daa7bd..aaa98fa7a6a3 100644 --- a/packages/rocketchat-file-upload/server/lib/FileUpload.js +++ b/packages/rocketchat-file-upload/server/lib/FileUpload.js @@ -1,6 +1,7 @@ /* globals UploadFS */ import fs from 'fs'; +import stream from 'stream'; import mime from 'mime-type/with-db'; import Future from 'fibers/future'; @@ -256,9 +257,33 @@ export class FileUploadClass { return this.delete(file._id); } - insert(file, stream, cb) { - const fileId = this.store.create(file); + insert(fileData, streamOrBuffer, cb) { + const fileId = this.store.create(fileData); + const token = this.store.createToken(fileId); + const tmpFile = UploadFS.getTempFilePath(fileId); - this.store.write(stream, fileId, cb); + try { + if (streamOrBuffer instanceof stream) { + stream.pipe(fs.createWriteStream(tmpFile)); + } else if (streamOrBuffer instanceof Buffer) { + fs.writeFileSync(tmpFile, streamOrBuffer); + } else { + throw new Error('Invalid file type'); + } + + const file = Meteor.call('ufsComplete', fileId, this.name, token); + + if (cb) { + cb(null, file); + } + + return file; + } catch (e) { + if (cb) { + cb(e); + } else { + throw e; + } + } } } diff --git a/packages/rocketchat-file-upload/ufs/AmazonS3/server.js b/packages/rocketchat-file-upload/ufs/AmazonS3/server.js index ab3332efb519..f69b05a883c4 100644 --- a/packages/rocketchat-file-upload/ufs/AmazonS3/server.js +++ b/packages/rocketchat-file-upload/ufs/AmazonS3/server.js @@ -63,6 +63,10 @@ export class AmazonS3Store extends UploadFS.Store { this.create = function(file, callback) { check(file, Object); + if (file._id == null) { + file._id = Random.id(); + } + file.AmazonS3 = { path: this.options.getPath(file) }; diff --git a/packages/rocketchat-file-upload/ufs/GoogleStorage/server.js b/packages/rocketchat-file-upload/ufs/GoogleStorage/server.js index 484f80a50d58..a6536d0a3342 100644 --- a/packages/rocketchat-file-upload/ufs/GoogleStorage/server.js +++ b/packages/rocketchat-file-upload/ufs/GoogleStorage/server.js @@ -48,6 +48,10 @@ export class GoogleStorageStore extends UploadFS.Store { this.create = function(file, callback) { check(file, Object); + if (file._id == null) { + file._id = Random.id(); + } + file.GoogleStorage = { path: this.options.getPath(file) }; diff --git a/packages/rocketchat-importer/server/classes/ImporterBase.coffee b/packages/rocketchat-importer/server/classes/ImporterBase.coffee index a7eff1e2752b..4a6eda164081 100644 --- a/packages/rocketchat-importer/server/classes/ImporterBase.coffee +++ b/packages/rocketchat-importer/server/classes/ImporterBase.coffee @@ -161,48 +161,45 @@ Importer.Base = class Importer.Base requestModule = if /https/i.test(fileUrl) then Importer.Base.https else Importer.Base.http requestModule.get fileUrl, Meteor.bindEnvironment((stream) -> - fileStore = UploadFS.getStore('Uploads') - fileId = fileStore.create details - if fileId - fileStore.write stream, fileId, (err, file) -> - if err - throw new Error(err) - else - url = file.url.replace(Meteor.absoluteUrl(), '/') - - attachment = - title: "File Uploaded: #{file.name}" - title_link: url - - if /^image\/.+/.test file.type - attachment.image_url = url - attachment.image_type = file.type - attachment.image_size = file.size - attachment.image_dimensions = file.identify?.size - - if /^audio\/.+/.test file.type - attachment.audio_url = url - attachment.audio_type = file.type - attachment.audio_size = file.size - - if /^video\/.+/.test file.type - attachment.video_url = url - attachment.video_type = file.type - attachment.video_size = file.size - - msg = - rid: details.rid - ts: timeStamp - msg: '' - file: - _id: file._id - groupable: false - attachments: [attachment] - - if details.message_id? and (typeof details.message_id is 'string') - msg['_id'] = details.message_id - - RocketChat.sendMessage user, msg, room, true - else - @logger.error "Failed to create the store for #{fileUrl}!!!" + fileStore = FileUpload.getStore('Uploads') + + fileStore.insert details, stream, (err, file) -> + if err + throw new Error(err) + else + url = file.url.replace(Meteor.absoluteUrl(), '/') + + attachment = + title: "File Uploaded: #{file.name}" + title_link: url + + if /^image\/.+/.test file.type + attachment.image_url = url + attachment.image_type = file.type + attachment.image_size = file.size + attachment.image_dimensions = file.identify?.size + + if /^audio\/.+/.test file.type + attachment.audio_url = url + attachment.audio_type = file.type + attachment.audio_size = file.size + + if /^video\/.+/.test file.type + attachment.video_url = url + attachment.video_type = file.type + attachment.video_size = file.size + + msg = + rid: details.rid + ts: timeStamp + msg: '' + file: + _id: file._id + groupable: false + attachments: [attachment] + + if details.message_id? and (typeof details.message_id is 'string') + msg['_id'] = details.message_id + + RocketChat.sendMessage user, msg, room, true ) diff --git a/packages/rocketchat-lib/server/functions/setUserAvatar.js b/packages/rocketchat-lib/server/functions/setUserAvatar.js index 760ef3deabb9..480688d7904b 100644 --- a/packages/rocketchat-lib/server/functions/setUserAvatar.js +++ b/packages/rocketchat-lib/server/functions/setUserAvatar.js @@ -39,16 +39,17 @@ RocketChat.setUserAvatar = function(user, dataURI, contentType, service) { contentType = fileData.contentType; } - const rs = RocketChatFile.bufferToStream(new Buffer(image, encoding)); + const buffer = new Buffer(image, encoding); const fileStore = FileUpload.getStore('Avatars'); fileStore.deleteByName(user.username); const file = { userId: user._id, - type: contentType + type: contentType, + size: buffer.length }; - fileStore.insert(file, rs, () => { + fileStore.insert(file, buffer, () => { Meteor.setTimeout(function() { RocketChat.models.Users.setAvatarOrigin(user._id, service); RocketChat.Notifications.notifyLogged('updateAvatar', {username: user.username}); diff --git a/packages/rocketchat-slackbridge/slackbridge.js b/packages/rocketchat-slackbridge/slackbridge.js index cacd3a3e9953..052f0cbafa73 100644 --- a/packages/rocketchat-slackbridge/slackbridge.js +++ b/packages/rocketchat-slackbridge/slackbridge.js @@ -592,59 +592,57 @@ class SlackBridge { const parsedUrl = url.parse(slackFileURL, true); parsedUrl.headers = { 'Authorization': `Bearer ${ this.apiToken }` }; requestModule.get(parsedUrl, Meteor.bindEnvironment((stream) => { - const fileStore = UploadFS.getStore('Uploads'); - const fileId = fileStore.create(details); - if (fileId) { - fileStore.write(stream, fileId, (err, file) => { - if (err) { - throw new Error(err); - } else { - const url = file.url.replace(Meteor.absoluteUrl(), '/'); - const attachment = { - title: `File Uploaded: ${ file.name }`, - title_link: url - }; - - if (/^image\/.+/.test(file.type)) { - attachment.image_url = url; - attachment.image_type = file.type; - attachment.image_size = file.size; - attachment.image_dimensions = file.identify && file.identify.size; - } - if (/^audio\/.+/.test(file.type)) { - attachment.audio_url = url; - attachment.audio_type = file.type; - attachment.audio_size = file.size; - } - if (/^video\/.+/.test(file.type)) { - attachment.video_url = url; - attachment.video_type = file.type; - attachment.video_size = file.size; - } + const fileStore = FileUpload.getStore('Uploads'); - const msg = { - rid: details.rid, - ts: timeStamp, - msg: '', - file: { - _id: file._id - }, - groupable: false, - attachments: [attachment] - }; + fileStore.insert(details, stream, (err, file) => { + if (err) { + throw new Error(err); + } else { + const url = file.url.replace(Meteor.absoluteUrl(), '/'); + const attachment = { + title: `File Uploaded: ${ file.name }`, + title_link: url + }; - if (isImporting) { - msg.imported = 'slackbridge'; - } + if (/^image\/.+/.test(file.type)) { + attachment.image_url = url; + attachment.image_type = file.type; + attachment.image_size = file.size; + attachment.image_dimensions = file.identify && file.identify.size; + } + if (/^audio\/.+/.test(file.type)) { + attachment.audio_url = url; + attachment.audio_type = file.type; + attachment.audio_size = file.size; + } + if (/^video\/.+/.test(file.type)) { + attachment.video_url = url; + attachment.video_type = file.type; + attachment.video_size = file.size; + } - if (details.message_id && (typeof details.message_id === 'string')) { - msg['_id'] = details.message_id; - } + const msg = { + rid: details.rid, + ts: timeStamp, + msg: '', + file: { + _id: file._id + }, + groupable: false, + attachments: [attachment] + }; - return RocketChat.sendMessage(rocketUser, msg, rocketChannel, true); + if (isImporting) { + msg.imported = 'slackbridge'; } - }); - } + + if (details.message_id && (typeof details.message_id === 'string')) { + msg['_id'] = details.message_id; + } + + return RocketChat.sendMessage(rocketUser, msg, rocketChannel, true); + } + }); })); } From a6aea83861e0226ecf83eea54700173f17339dc9 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Mon, 29 May 2017 10:55:55 -0300 Subject: [PATCH 31/42] Fix ESLint errors --- packages/rocketchat-slackbridge/slackbridge.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rocketchat-slackbridge/slackbridge.js b/packages/rocketchat-slackbridge/slackbridge.js index 052f0cbafa73..e48446794082 100644 --- a/packages/rocketchat-slackbridge/slackbridge.js +++ b/packages/rocketchat-slackbridge/slackbridge.js @@ -1,4 +1,4 @@ -/* globals logger, UploadFS */ +/* globals logger */ class SlackBridge { From 6eb06e6cdb8d4fa2157d00e01b4ff3522d1ab8bd Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Mon, 29 May 2017 12:06:23 -0300 Subject: [PATCH 32/42] Uploads: Delete from correct store --- .../rocketchat-file-upload/server/lib/FileUpload.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/rocketchat-file-upload/server/lib/FileUpload.js b/packages/rocketchat-file-upload/server/lib/FileUpload.js index aaa98fa7a6a3..cca5a0d60105 100644 --- a/packages/rocketchat-file-upload/server/lib/FileUpload.js +++ b/packages/rocketchat-file-upload/server/lib/FileUpload.js @@ -176,6 +176,10 @@ Object.assign(FileUpload, { const storageType = RocketChat.settings.get('FileUpload_Storage_Type'); const handlerName = `${ storageType }:${ modelName }`; + return this.getStoreByName(handlerName); + }, + + getStoreByName(handlerName) { if (this.handlers[handlerName] == null) { console.error(`Upload handler "${ handlerName }" does not exists`); } @@ -244,7 +248,9 @@ export class FileUploadClass { return; } - return this.delete(file._id); + const store = FileUpload.getStoreByName(file.store); + + return store.delete(file._id); } deleteByName(fileName) { @@ -254,7 +260,9 @@ export class FileUploadClass { return; } - return this.delete(file._id); + const store = FileUpload.getStoreByName(file.store); + + return store.delete(file._id); } insert(fileData, streamOrBuffer, cb) { From b29b03d906e666bd96adc18ece9e3f6fc48f2efd Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Mon, 29 May 2017 15:23:44 -0300 Subject: [PATCH 33/42] Remove RocketChatFileAvatarInstance from eslintrc --- .eslintrc | 1 - 1 file changed, 1 deletion(-) diff --git a/.eslintrc b/.eslintrc index b8b913dd91b6..6e652c53b845 100644 --- a/.eslintrc +++ b/.eslintrc @@ -133,7 +133,6 @@ "ReactiveVar" : false, "RocketChat" : true, "RocketChatFile" : false, - "RocketChatFileAvatarInstance": false, "RoomHistoryManager" : false, "RoomManager" : false, "s" : false, From d0735ac9adf5e10c4008b01c5704838b4a411442 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Mon, 29 May 2017 16:01:49 -0300 Subject: [PATCH 34/42] Uploads: Migrate old avatars --- server/startup/migrations/v096.js | 60 +++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/server/startup/migrations/v096.js b/server/startup/migrations/v096.js index 6016a3899fc9..60d838169c7a 100644 --- a/server/startup/migrations/v096.js +++ b/server/startup/migrations/v096.js @@ -1,3 +1,6 @@ +import fs from 'fs'; +import path from 'path'; + RocketChat.Migrations.add({ version: 96, up() { @@ -59,5 +62,62 @@ RocketChat.Migrations.add({ }, { multi: true }); + + const avatarOrigins = [ + 'upload', + 'gravatar', + 'facebook', + 'twitter', + 'github', + 'google', + 'url', + 'gitlab', + 'linkedin' + ]; + + const avatarsFileStore = FileUpload.getStore('Avatars'); + + const oldAvatarGridFS = new RocketChatFile.GridFS({ + name: 'avatars' + }); + + + RocketChat.models.Users.find({avatarOrigin: {$in: avatarOrigins}}, {avatarOrigin: 1, username: 1}).forEach((user) => { + const id = `${ user.username }.jpg`; + const gridFSAvatar = oldAvatarGridFS.getFileWithReadStream(id); + + if (gridFSAvatar) { + const details = { + userId: user._id, + type: gridFSAvatar.contentType, + size: gridFSAvatar.length + }; + + avatarsFileStore.insert(details, gridFSAvatar.readStream, () => { + oldAvatarGridFS.deleteFile(id); + }); + } else { + const avatarsPath = RocketChat.settings.get('Accounts_AvatarStorePath'); + if (avatarsPath && avatarsPath.trim()) { + const filePath = path.join(avatarsPath, id); + const stat = fs.statSync(filePath); + if (stat && stat.isFile()) { + const rs = fs.createReadStream(filePath); + const details = { + userId: user._id, + type: 'image/jpeg', + size: stat.size + }; + + avatarsFileStore.insert(details, rs, () => { + fs.unlinkSync(filePath); + }); + } + } + } + }); + + RocketChat.models.Settings.remove({_id: 'Accounts_AvatarStoreType'}); + RocketChat.models.Settings.remove({_id: 'Accounts_AvatarStorePath'}); } }); From f3108d72bc95695a43f719840a390a4d5cf96a8b Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Mon, 29 May 2017 16:01:57 -0300 Subject: [PATCH 35/42] Remove old avatar settings --- .../server/config/configFileUploadAmazonS3.js | 2 +- .../config/configFileUploadFileSystem.js | 2 +- .../config/configFileUploadGoogleStorage.js | 2 +- packages/rocketchat-i18n/i18n/ar.i18n.json | 2 -- packages/rocketchat-i18n/i18n/ca.i18n.json | 2 -- packages/rocketchat-i18n/i18n/cs.i18n.json | 2 -- packages/rocketchat-i18n/i18n/de-AT.i18n.json | 2 -- packages/rocketchat-i18n/i18n/de.i18n.json | 2 -- packages/rocketchat-i18n/i18n/el.i18n.json | 2 -- packages/rocketchat-i18n/i18n/en.i18n.json | 2 -- packages/rocketchat-i18n/i18n/es.i18n.json | 2 -- packages/rocketchat-i18n/i18n/fa.i18n.json | 2 -- packages/rocketchat-i18n/i18n/fi.i18n.json | 2 -- packages/rocketchat-i18n/i18n/fr.i18n.json | 2 -- packages/rocketchat-i18n/i18n/he.i18n.json | 2 -- packages/rocketchat-i18n/i18n/hr.i18n.json | 2 -- packages/rocketchat-i18n/i18n/hu.i18n.json | 2 -- packages/rocketchat-i18n/i18n/id.i18n.json | 2 -- packages/rocketchat-i18n/i18n/it.i18n.json | 2 -- packages/rocketchat-i18n/i18n/ja.i18n.json | 2 -- packages/rocketchat-i18n/i18n/km.i18n.json | 2 -- packages/rocketchat-i18n/i18n/ko.i18n.json | 2 -- packages/rocketchat-i18n/i18n/ku.i18n.json | 2 -- packages/rocketchat-i18n/i18n/lo.i18n.json | 2 -- packages/rocketchat-i18n/i18n/ms-MY.i18n.json | 2 -- packages/rocketchat-i18n/i18n/nl.i18n.json | 2 -- packages/rocketchat-i18n/i18n/pl.i18n.json | 2 -- packages/rocketchat-i18n/i18n/pt-BR.i18n.json | 2 -- packages/rocketchat-i18n/i18n/pt.i18n.json | 2 -- packages/rocketchat-i18n/i18n/ro.i18n.json | 2 -- packages/rocketchat-i18n/i18n/ru.i18n.json | 2 -- packages/rocketchat-i18n/i18n/sq.i18n.json | 2 -- packages/rocketchat-i18n/i18n/sr.i18n.json | 2 -- packages/rocketchat-i18n/i18n/sv.i18n.json | 2 -- packages/rocketchat-i18n/i18n/ta-IN.i18n.json | 2 -- packages/rocketchat-i18n/i18n/tr.i18n.json | 2 -- packages/rocketchat-i18n/i18n/ug.i18n.json | 2 -- packages/rocketchat-i18n/i18n/uk.i18n.json | 2 -- packages/rocketchat-i18n/i18n/zh-HK.i18n.json | 1 - packages/rocketchat-i18n/i18n/zh-TW.i18n.json | 2 -- packages/rocketchat-i18n/i18n/zh.i18n.json | 2 -- .../rocketchat-lib/server/startup/settings.js | 20 +------------------ 42 files changed, 4 insertions(+), 97 deletions(-) diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js index b6ba1651dfc4..0f98c4a9e7d3 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js @@ -1,4 +1,4 @@ -/* globals FileUpload, RocketChatFile */ +/* globals FileUpload */ import { FileUploadClass } from '../lib/FileUpload'; import '../../ufs/AmazonS3/server.js'; diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js b/packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js index 5469c6382070..263baad99078 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js @@ -1,4 +1,4 @@ -/* globals FileUpload, UploadFS, RocketChatFile */ +/* globals FileUpload, UploadFS */ import fs from 'fs'; import { FileUploadClass } from '../lib/FileUpload'; diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js index 154ba89e63e0..67fcc6b75c34 100644 --- a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js +++ b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js @@ -1,4 +1,4 @@ -/* globals FileUpload, RocketChatFile */ +/* globals FileUpload */ import { FileUploadClass } from '../lib/FileUpload'; import '../../ufs/GoogleStorage/server.js'; diff --git a/packages/rocketchat-i18n/i18n/ar.i18n.json b/packages/rocketchat-i18n/i18n/ar.i18n.json index 3dbd8af238ac..5e4f6947acbb 100644 --- a/packages/rocketchat-i18n/i18n/ar.i18n.json +++ b/packages/rocketchat-i18n/i18n/ar.i18n.json @@ -21,8 +21,6 @@ "Accounts_AllowUserProfileChange": "السماح بتعديل الملف الشخصي للعضو", "Accounts_AvatarResize": "تغيير حجم الصور الرمزية", "Accounts_AvatarSize": "حجم الصورة الرمزية", - "Accounts_AvatarStorePath": "مسار تخزين الصورة الرمزية", - "Accounts_AvatarStoreType": "نوع تخزين الصورة الرمزية", "Accounts_BlockedDomainsList": "محظور قائمة المجالات", "Accounts_BlockedDomainsList_Description": "قائمة مفصولة بفواصل من المجالات سدت", "Accounts_BlockedUsernameList": "قائمة اﻷسماء المحظورة", diff --git a/packages/rocketchat-i18n/i18n/ca.i18n.json b/packages/rocketchat-i18n/i18n/ca.i18n.json index 185c793f6340..d9b22df36000 100644 --- a/packages/rocketchat-i18n/i18n/ca.i18n.json +++ b/packages/rocketchat-i18n/i18n/ca.i18n.json @@ -29,8 +29,6 @@ "Accounts_AllowUserProfileChange": "Permetre modificar el perfil d'usuari", "Accounts_AvatarResize": "Canviar la mida dels avatars", "Accounts_AvatarSize": "Mida d'avatar", - "Accounts_AvatarStorePath": "Ruta d'emmagatzematge dels avatars", - "Accounts_AvatarStoreType": "Tipus d'emmagatzematge dels avatars", "Accounts_BlockedDomainsList": "Llista de dominis bloquejats", "Accounts_BlockedDomainsList_Description": "Llista de dominis bloquejats separada per comes", "Accounts_BlockedUsernameList": "Llista de noms d'usuari bloquejats", diff --git a/packages/rocketchat-i18n/i18n/cs.i18n.json b/packages/rocketchat-i18n/i18n/cs.i18n.json index f3d89cf4c104..81eba2d08ce2 100644 --- a/packages/rocketchat-i18n/i18n/cs.i18n.json +++ b/packages/rocketchat-i18n/i18n/cs.i18n.json @@ -29,8 +29,6 @@ "Accounts_AllowUserProfileChange": "Povolit úpravy profilu", "Accounts_AvatarResize": "Rozměry avataru", "Accounts_AvatarSize": "Velikost avataru", - "Accounts_AvatarStorePath": "Cesta k úložišti avatarů", - "Accounts_AvatarStoreType": "Typ úložiště avatarů", "Accounts_BlockedDomainsList": "Seznam blokovaných domén", "Accounts_BlockedDomainsList_Description": "Čárkami oddělený seznam blokovaných domén", "Accounts_BlockedUsernameList": "Zakázaná uživatelská jména", diff --git a/packages/rocketchat-i18n/i18n/de-AT.i18n.json b/packages/rocketchat-i18n/i18n/de-AT.i18n.json index f89e6d847474..be0363e99f67 100644 --- a/packages/rocketchat-i18n/i18n/de-AT.i18n.json +++ b/packages/rocketchat-i18n/i18n/de-AT.i18n.json @@ -27,8 +27,6 @@ "Accounts_AllowUserProfileChange": "Benutzern das Ändern des Profils erlauben", "Accounts_AvatarResize": "Größe des Profilbilds anpassen", "Accounts_AvatarSize": "Größe des Profilbilds", - "Accounts_AvatarStorePath": "Speicherpfad des Profilbilds", - "Accounts_AvatarStoreType": "Speichertyp des Profilbilds", "Accounts_BlockedDomainsList": "Liste geblockter Domains", "Accounts_BlockedDomainsList_Description": "Kommata getrennte Liste von geblockten Domains", "Accounts_BlockedUsernameList": "Liste gesperrter Benutzernamen", diff --git a/packages/rocketchat-i18n/i18n/de.i18n.json b/packages/rocketchat-i18n/i18n/de.i18n.json index 8503b587ac36..3d3df2d00b20 100644 --- a/packages/rocketchat-i18n/i18n/de.i18n.json +++ b/packages/rocketchat-i18n/i18n/de.i18n.json @@ -29,8 +29,6 @@ "Accounts_AllowUserProfileChange": "Benutzern das Ändern des Profils erlauben", "Accounts_AvatarResize": "Größe des Profilbilds anpassen", "Accounts_AvatarSize": "Größe des Profilbilds", - "Accounts_AvatarStorePath": "Speicherpfad des Profilbilds", - "Accounts_AvatarStoreType": "Speichertyp des Profilbilds", "Accounts_BlockedDomainsList": "Liste geblockter Domains", "Accounts_BlockedDomainsList_Description": "Kommata getrennte Liste von geblockten Domains", "Accounts_BlockedUsernameList": "Liste gesperrter Benutzernamen", diff --git a/packages/rocketchat-i18n/i18n/el.i18n.json b/packages/rocketchat-i18n/i18n/el.i18n.json index d4046ac1362d..ba47065b0679 100644 --- a/packages/rocketchat-i18n/i18n/el.i18n.json +++ b/packages/rocketchat-i18n/i18n/el.i18n.json @@ -21,8 +21,6 @@ "Accounts_AllowUserProfileChange": "Επιτρέψτε Προφίλ Χρήστη Αλλαγή", "Accounts_AvatarResize": "Αλλαγή μεγέθους Avatars", "Accounts_AvatarSize": "Avatar Μέγεθος", - "Accounts_AvatarStorePath": "Path Avatar αποθήκευσης", - "Accounts_AvatarStoreType": "Avatar Τύπος αποθήκευσης", "Accounts_BlockedDomainsList": "Λίστα αποκλεισμένων τομέων", "Accounts_BlockedDomainsList_Description": "Διαχωρισμένες με κόμμα λίστα των αποκλεισμένων περιοχών", "Accounts_BlockedUsernameList": "Λίστα αποκλεισμένων Όνομα Χρήστη", diff --git a/packages/rocketchat-i18n/i18n/en.i18n.json b/packages/rocketchat-i18n/i18n/en.i18n.json index 0be9c641d9cf..e2b66fd247cb 100644 --- a/packages/rocketchat-i18n/i18n/en.i18n.json +++ b/packages/rocketchat-i18n/i18n/en.i18n.json @@ -29,8 +29,6 @@ "Accounts_AllowUserProfileChange": "Allow User Profile Change", "Accounts_AvatarResize": "Resize Avatars", "Accounts_AvatarSize": "Avatar Size", - "Accounts_AvatarStorePath": "Avatar Storage Path", - "Accounts_AvatarStoreType": "Avatar Storage Type", "Accounts_BlockedDomainsList": "Blocked Domains List", "Accounts_BlockedDomainsList_Description": "Comma-separated list of blocked domains", "Accounts_BlockedUsernameList": "Blocked Username List", diff --git a/packages/rocketchat-i18n/i18n/es.i18n.json b/packages/rocketchat-i18n/i18n/es.i18n.json index e5dceb0f4260..915652d16149 100644 --- a/packages/rocketchat-i18n/i18n/es.i18n.json +++ b/packages/rocketchat-i18n/i18n/es.i18n.json @@ -27,8 +27,6 @@ "Accounts_AllowUserProfileChange": "Permitir al Usuario modificar su Perfil", "Accounts_AvatarResize": "Cambiar el Tamaño de los Avatars", "Accounts_AvatarSize": "Tamaño de Avatar", - "Accounts_AvatarStorePath": "Ruta de almacenamiento de los avatares", - "Accounts_AvatarStoreType": "Tipo de Almacenamiento de Avatar", "Accounts_BlockedDomainsList": "Lista de dominios bloqueados", "Accounts_BlockedDomainsList_Description": "Lista de dominios bloqueados separada por comas", "Accounts_BlockedUsernameList": "Lista de nombres de usuario bloqueados", diff --git a/packages/rocketchat-i18n/i18n/fa.i18n.json b/packages/rocketchat-i18n/i18n/fa.i18n.json index 040e0fbb6974..db2d3e05f9a7 100644 --- a/packages/rocketchat-i18n/i18n/fa.i18n.json +++ b/packages/rocketchat-i18n/i18n/fa.i18n.json @@ -21,8 +21,6 @@ "Accounts_AllowUserProfileChange": "کاربر اجازه می دهد مشخصات تغییر", "Accounts_AvatarResize": "تغییر اندازه آواتار ها", "Accounts_AvatarSize": "آواتار حجم", - "Accounts_AvatarStorePath": "مسیر آواتار ذخیره سازی", - "Accounts_AvatarStoreType": "آواتار نوع ذخیره سازی", "Accounts_BlockedDomainsList": "مسدود شده فهرست دامنه", "Accounts_BlockedDomainsList_Description": "جدا شده با کاما از حوزه های مسدود شده", "Accounts_BlockedUsernameList": "مسدود شده نام کاربری", diff --git a/packages/rocketchat-i18n/i18n/fi.i18n.json b/packages/rocketchat-i18n/i18n/fi.i18n.json index 2b0f459d43b8..fbfdaa6bc411 100644 --- a/packages/rocketchat-i18n/i18n/fi.i18n.json +++ b/packages/rocketchat-i18n/i18n/fi.i18n.json @@ -26,8 +26,6 @@ "Accounts_AllowUserProfileChange": "Salli käyttäjän profiilin muutos", "Accounts_AvatarResize": "Muuta avatarien kokoa", "Accounts_AvatarSize": "Avatarin koko", - "Accounts_AvatarStorePath": "Avatarin tallennuspolku", - "Accounts_AvatarStoreType": "Avatarien tallennusmuoto", "Accounts_BlockedDomainsList": "Estettyjen verkkotunnusten lista", "Accounts_BlockedDomainsList_Description": "Pilkuilla eroteltu lista estetyistä verkkotunnuksista", "Accounts_BlockedUsernameList": "Estettyjen käyttäjätunnusten lista", diff --git a/packages/rocketchat-i18n/i18n/fr.i18n.json b/packages/rocketchat-i18n/i18n/fr.i18n.json index 96b29850cf85..e3944df550d2 100644 --- a/packages/rocketchat-i18n/i18n/fr.i18n.json +++ b/packages/rocketchat-i18n/i18n/fr.i18n.json @@ -29,8 +29,6 @@ "Accounts_AllowUserProfileChange": "Autoriser la modification de profil", "Accounts_AvatarResize": "Redimensionner les avatars", "Accounts_AvatarSize": "Taille de l'avatar", - "Accounts_AvatarStorePath": "Chemin de stockage des avatars", - "Accounts_AvatarStoreType": "Type de stockage des avatars", "Accounts_BlockedDomainsList": "Liste des domaines bloqués", "Accounts_BlockedDomainsList_Description": "Liste de domaines bloqués, séparés par des virgules", "Accounts_BlockedUsernameList": "Liste des noms d'utilisateurs bloqués", diff --git a/packages/rocketchat-i18n/i18n/he.i18n.json b/packages/rocketchat-i18n/i18n/he.i18n.json index b912f90ea58d..3be918e8545b 100644 --- a/packages/rocketchat-i18n/i18n/he.i18n.json +++ b/packages/rocketchat-i18n/i18n/he.i18n.json @@ -21,8 +21,6 @@ "Accounts_AllowUserProfileChange": "אפשר למשתמש לשנות את הפרופיל", "Accounts_AvatarResize": "שנה את גודל תמונת הפרופיל", "Accounts_AvatarSize": "גודל תמונת הפרופיל", - "Accounts_AvatarStorePath": "נתיב תמונת הפרופיל", - "Accounts_AvatarStoreType": "סוג אחסון תמונת פרופיל", "Accounts_BlockedDomainsList": "רשימת דומיינים חסומים", "Accounts_BlockedDomainsList_Description": "רשימה מופרדת בפסיקים של דומיינים חסומים", "Accounts_BlockedUsernameList": "רשימת משתמש חסום", diff --git a/packages/rocketchat-i18n/i18n/hr.i18n.json b/packages/rocketchat-i18n/i18n/hr.i18n.json index 83e9f1a937d0..56c2306b8c87 100644 --- a/packages/rocketchat-i18n/i18n/hr.i18n.json +++ b/packages/rocketchat-i18n/i18n/hr.i18n.json @@ -27,8 +27,6 @@ "Accounts_AllowUserProfileChange": "Dopusti Promjenu Profila", "Accounts_AvatarResize": "Promjeni veličinu Avatara", "Accounts_AvatarSize": "Veličina Avatara", - "Accounts_AvatarStorePath": "Putanja do spremišta Avatara ", - "Accounts_AvatarStoreType": "Tip Spremišta Avatara", "Accounts_BlockedDomainsList": "Popis blokiranih domena", "Accounts_BlockedDomainsList_Description": "Popis blokiranih domena odvojenih zarezom", "Accounts_BlockedUsernameList": "Popis blokiranih korisničkih imena", diff --git a/packages/rocketchat-i18n/i18n/hu.i18n.json b/packages/rocketchat-i18n/i18n/hu.i18n.json index 93d556692db4..7a936671ff20 100644 --- a/packages/rocketchat-i18n/i18n/hu.i18n.json +++ b/packages/rocketchat-i18n/i18n/hu.i18n.json @@ -27,8 +27,6 @@ "Accounts_AllowUserProfileChange": "Felhasználó módosíthatja profilját", "Accounts_AvatarResize": "Profilképek átméretezése", "Accounts_AvatarSize": "Profilkép mérete", - "Accounts_AvatarStorePath": "Profilkép tárolásának elérési útja", - "Accounts_AvatarStoreType": "Profilkép tárolásának típusa", "Accounts_BlockedDomainsList": "Blokkolt domainek listája", "Accounts_BlockedDomainsList_Description": "Blokkolt domain-ek listája (vesszővel elválasztva)", "Accounts_BlockedUsernameList": "Blokkolt felhasználónevek listája", diff --git a/packages/rocketchat-i18n/i18n/id.i18n.json b/packages/rocketchat-i18n/i18n/id.i18n.json index c41f59d8ca82..954bc6247aba 100644 --- a/packages/rocketchat-i18n/i18n/id.i18n.json +++ b/packages/rocketchat-i18n/i18n/id.i18n.json @@ -21,8 +21,6 @@ "Accounts_AllowUserProfileChange": "Memungkinkan Profil Pengguna Ganti", "Accounts_AvatarResize": "Ubah Ukuran Avatar", "Accounts_AvatarSize": "Ukuran Avatar", - "Accounts_AvatarStorePath": "Lokasi Penyimpanan Avatar", - "Accounts_AvatarStoreType": "Tipe Penyimpanan Avatar", "Accounts_BlockedDomainsList": "Daftar Domain diblokir", "Accounts_BlockedDomainsList_Description": "Dipisahkan dengan koma daftar domain diblokir", "Accounts_BlockedUsernameList": "Diblokir Daftar Nama", diff --git a/packages/rocketchat-i18n/i18n/it.i18n.json b/packages/rocketchat-i18n/i18n/it.i18n.json index 06d62a2f4d7f..f7d280bdb339 100644 --- a/packages/rocketchat-i18n/i18n/it.i18n.json +++ b/packages/rocketchat-i18n/i18n/it.i18n.json @@ -27,8 +27,6 @@ "Accounts_AllowUserProfileChange": "Consenti cambio profilo utente", "Accounts_AvatarResize": "Ridimensiona Avatar", "Accounts_AvatarSize": "Dimensione Avatar", - "Accounts_AvatarStorePath": "Percorso Avatar", - "Accounts_AvatarStoreType": "Tipo di archiviazione per gli Avatar", "Accounts_BlockedDomainsList": "Elenco domini bloccati", "Accounts_BlockedDomainsList_Description": "elenco separato da virgole dei domini bloccati", "Accounts_BlockedUsernameList": "Lista nomi utente bloccati", diff --git a/packages/rocketchat-i18n/i18n/ja.i18n.json b/packages/rocketchat-i18n/i18n/ja.i18n.json index d207cdff4f02..39155bf24657 100644 --- a/packages/rocketchat-i18n/i18n/ja.i18n.json +++ b/packages/rocketchat-i18n/i18n/ja.i18n.json @@ -21,8 +21,6 @@ "Accounts_AllowUserProfileChange": "プロフィールの変更を許可する", "Accounts_AvatarResize": "アバターの大きさを変更する", "Accounts_AvatarSize": "アバターの大きさ", - "Accounts_AvatarStorePath": "アバターの保存先", - "Accounts_AvatarStoreType": "アバターの保存先ストレージ種類", "Accounts_BlockedDomainsList": "ブロックされたドメイン一覧", "Accounts_BlockedDomainsList_Description": "カンマ区切りのブロックされたドメイン一覧", "Accounts_BlockedUsernameList": "ブロックされたユーザー名の一覧", diff --git a/packages/rocketchat-i18n/i18n/km.i18n.json b/packages/rocketchat-i18n/i18n/km.i18n.json index 2374ff4d1139..5a6446815f25 100644 --- a/packages/rocketchat-i18n/i18n/km.i18n.json +++ b/packages/rocketchat-i18n/i18n/km.i18n.json @@ -21,8 +21,6 @@ "Accounts_AllowUserProfileChange": "អនុញ្ញាតិអ្នកប្រើប្រាស់ប្តូរព័ត៌មានផ្ទាល់ខ្លួន", "Accounts_AvatarResize": "ប្តូ​រ​ទំហំ Avatar", "Accounts_AvatarSize": "ទំហំ Avatar", - "Accounts_AvatarStorePath": "ទីតាំងផ្ទុក Avatar", - "Accounts_AvatarStoreType": "ប្រភេទបន្ទុក Avatar", "Accounts_BlockedDomainsList": "បញ្ជីដែន", "Accounts_BlockedDomainsList_Description": "បញ្ជីបំបែកដោយសញ្ញាក្បៀសនៃដែនបានបិទ", "Accounts_BlockedUsernameList": "បញ្ជីឈ្មោះអ្នកប្រើ", diff --git a/packages/rocketchat-i18n/i18n/ko.i18n.json b/packages/rocketchat-i18n/i18n/ko.i18n.json index c02eee6cb01b..5d7f599dd39c 100644 --- a/packages/rocketchat-i18n/i18n/ko.i18n.json +++ b/packages/rocketchat-i18n/i18n/ko.i18n.json @@ -26,8 +26,6 @@ "Accounts_AllowUserProfileChange": "사용자 프로필 변경을 허용", "Accounts_AvatarResize": "아바타 크기 조정", "Accounts_AvatarSize": "아바타 크기", - "Accounts_AvatarStorePath": "아바타 저장 경로", - "Accounts_AvatarStoreType": "아바파 저장 타입", "Accounts_BlockedDomainsList": "차단된 도메인 목록", "Accounts_BlockedDomainsList_Description": "쉼표로 구문된 차단 도메인 리스트", "Accounts_BlockedUsernameList": "차단된 사용자 리스트", diff --git a/packages/rocketchat-i18n/i18n/ku.i18n.json b/packages/rocketchat-i18n/i18n/ku.i18n.json index ac6d31aede77..b4fd33eb3a3e 100644 --- a/packages/rocketchat-i18n/i18n/ku.i18n.json +++ b/packages/rocketchat-i18n/i18n/ku.i18n.json @@ -21,8 +21,6 @@ "Accounts_AllowUserProfileChange": "Destûrê bide User Profile Change", "Accounts_AvatarResize": "resize Avatars", "Accounts_AvatarSize": "Avatar Size", - "Accounts_AvatarStorePath": "Path Avatar Storage", - "Accounts_AvatarStoreType": "Avatar Type Storage", "Accounts_BlockedDomainsList": "Astengkirin Lîsteya Domain", "Accounts_BlockedDomainsList_Description": "lîsteya bêhnok-cuda ji qada astengkirin", "Accounts_BlockedUsernameList": "Astengkirin Lîsteya Username", diff --git a/packages/rocketchat-i18n/i18n/lo.i18n.json b/packages/rocketchat-i18n/i18n/lo.i18n.json index 6c71bb90512d..158e1b799ce2 100644 --- a/packages/rocketchat-i18n/i18n/lo.i18n.json +++ b/packages/rocketchat-i18n/i18n/lo.i18n.json @@ -21,8 +21,6 @@ "Accounts_AllowUserProfileChange": "ອະນຸຍາດໃຫ້ການປ່ຽນແປງຂໍ້ມູນຂອງຜູ້ໃຊ້", "Accounts_AvatarResize": "ປັບຂະຫນາດນົດ", "Accounts_AvatarSize": "Avatar ຂະຫນາດ", - "Accounts_AvatarStorePath": "ເສັ້ນທາງກ້າວສູ່ຮູບສ່ວນຕົວການເກັບຮັກສາ", - "Accounts_AvatarStoreType": "Avatar ປະເພດການເກັບຮັກສາ", "Accounts_BlockedDomainsList": "ສະກັດຊີ Domains", "Accounts_BlockedDomainsList_Description": "ບັນຊີລາຍຊື່ຈຸດ, ແຍກຂອງໂດເມນກັດ", "Accounts_BlockedUsernameList": "ບັນຊີ Username ກັດ", diff --git a/packages/rocketchat-i18n/i18n/ms-MY.i18n.json b/packages/rocketchat-i18n/i18n/ms-MY.i18n.json index dfb52d0a5e78..0aa1d7eaf490 100644 --- a/packages/rocketchat-i18n/i18n/ms-MY.i18n.json +++ b/packages/rocketchat-i18n/i18n/ms-MY.i18n.json @@ -21,8 +21,6 @@ "Accounts_AllowUserProfileChange": "Benarkan Profil Pengguna Tukar", "Accounts_AvatarResize": "Saiz semula Avatars", "Accounts_AvatarSize": "avatar Saiz", - "Accounts_AvatarStorePath": "Avatar Storage Path", - "Accounts_AvatarStoreType": "Avatar Storage Jenis", "Accounts_BlockedDomainsList": "Senarai Domain Disekat", "Accounts_BlockedDomainsList_Description": "Senarai diasingkan koma bagi domain disekat", "Accounts_BlockedUsernameList": "Senarai Nama pengguna Disekat", diff --git a/packages/rocketchat-i18n/i18n/nl.i18n.json b/packages/rocketchat-i18n/i18n/nl.i18n.json index a175e83d6164..81f9d3fdeafb 100644 --- a/packages/rocketchat-i18n/i18n/nl.i18n.json +++ b/packages/rocketchat-i18n/i18n/nl.i18n.json @@ -21,8 +21,6 @@ "Accounts_AllowUserProfileChange": "Sta wijzen van gebruikers profiel toe", "Accounts_AvatarResize": "Wijzig Avatar grootte", "Accounts_AvatarSize": "Avatar grootte", - "Accounts_AvatarStorePath": "Avatar opslag pad", - "Accounts_AvatarStoreType": "Avatar opslag type", "Accounts_BlockedDomainsList": "Geblokkeerde domeinen List", "Accounts_BlockedDomainsList_Description": "Door komma's gescheiden lijst van geblokkeerde domeinen", "Accounts_BlockedUsernameList": "Geblokkeerde Gebruikersnaam List", diff --git a/packages/rocketchat-i18n/i18n/pl.i18n.json b/packages/rocketchat-i18n/i18n/pl.i18n.json index 28492fd71c39..c93dd52f309e 100644 --- a/packages/rocketchat-i18n/i18n/pl.i18n.json +++ b/packages/rocketchat-i18n/i18n/pl.i18n.json @@ -24,8 +24,6 @@ "Accounts_AllowUserProfileChange": "Pozwól na zmienianie profilów użytkowników", "Accounts_AvatarResize": "Zmiana rozmiaru avatarów", "Accounts_AvatarSize": "Rozmiar avataru", - "Accounts_AvatarStorePath": "Ścieżka przechowywania avatarów", - "Accounts_AvatarStoreType": "Rodzaj magazynu avatarów", "Accounts_BlockedDomainsList": "Lista zablokowanych domen", "Accounts_BlockedDomainsList_Description": "Oddzielonych przecinkami lista zablokowanych domen", "Accounts_BlockedUsernameList": "Lista zablokowanych użytkowników", diff --git a/packages/rocketchat-i18n/i18n/pt-BR.i18n.json b/packages/rocketchat-i18n/i18n/pt-BR.i18n.json index 6261000c036a..f34f587d3c20 100644 --- a/packages/rocketchat-i18n/i18n/pt-BR.i18n.json +++ b/packages/rocketchat-i18n/i18n/pt-BR.i18n.json @@ -27,8 +27,6 @@ "Accounts_AllowUserProfileChange": "Permitir que o usuário altere o perfil", "Accounts_AvatarResize": "Redimensionar Avatares", "Accounts_AvatarSize": "Tamanho do Avatar", - "Accounts_AvatarStorePath": "Caminho para armazenar Avatares", - "Accounts_AvatarStoreType": "Tipo de armazenamento de Avatares", "Accounts_BlockedDomainsList": "Lista de Domínios Bloqueados", "Accounts_BlockedDomainsList_Description": "Lista de domínios bloqueados, separados por vírgulas ", "Accounts_BlockedUsernameList": "Lista de nomes de usuário bloqueados", diff --git a/packages/rocketchat-i18n/i18n/pt.i18n.json b/packages/rocketchat-i18n/i18n/pt.i18n.json index 5e5098430a62..bcec56545ecb 100644 --- a/packages/rocketchat-i18n/i18n/pt.i18n.json +++ b/packages/rocketchat-i18n/i18n/pt.i18n.json @@ -29,8 +29,6 @@ "Accounts_AllowUserProfileChange": "Permitir que o usuário altere o perfil", "Accounts_AvatarResize": "Redimensionar Avatares", "Accounts_AvatarSize": "Tamanho do Avatar", - "Accounts_AvatarStorePath": "Caminho para armazenar Avatares", - "Accounts_AvatarStoreType": "Tipo de armazenamento de Avatares", "Accounts_BlockedDomainsList": "Lista de Domínios Bloqueados", "Accounts_BlockedDomainsList_Description": "Lista de domínios bloqueados, separados por vírgulas ", "Accounts_BlockedUsernameList": "Lista de nomes de usuário bloqueados", diff --git a/packages/rocketchat-i18n/i18n/ro.i18n.json b/packages/rocketchat-i18n/i18n/ro.i18n.json index 94f08574b0e6..1b9eba35a793 100644 --- a/packages/rocketchat-i18n/i18n/ro.i18n.json +++ b/packages/rocketchat-i18n/i18n/ro.i18n.json @@ -21,8 +21,6 @@ "Accounts_AllowUserProfileChange": "Permite schimbarea profilului utilzatorilor", "Accounts_AvatarResize": "Redimensionarea Avatare", "Accounts_AvatarSize": "Dimensiune Avatar", - "Accounts_AvatarStorePath": "Calea de stocare Avatar", - "Accounts_AvatarStoreType": "Tip stocare Avatar", "Accounts_BlockedDomainsList": "Blocate List Domenii", "Accounts_BlockedDomainsList_Description": "Lista de elemente separate prin virgulă de domenii blocate", "Accounts_BlockedUsernameList": "Utilizator blocat Lista", diff --git a/packages/rocketchat-i18n/i18n/ru.i18n.json b/packages/rocketchat-i18n/i18n/ru.i18n.json index 37ecf89f65ff..a8200eb61e4c 100644 --- a/packages/rocketchat-i18n/i18n/ru.i18n.json +++ b/packages/rocketchat-i18n/i18n/ru.i18n.json @@ -26,8 +26,6 @@ "Accounts_AllowUserProfileChange": "Разрешить пользователю изменять настройки профиля", "Accounts_AvatarResize": "Изменение размера аватара", "Accounts_AvatarSize": "Размер аватара", - "Accounts_AvatarStorePath": "Путь к хранилищу аватаров", - "Accounts_AvatarStoreType": "Тип хранилища аватаров", "Accounts_BlockedDomainsList": "Список запрещённых доменов", "Accounts_BlockedDomainsList_Description": "Список запрещённых доменов, разделенных запятой", "Accounts_BlockedUsernameList": "Список заблокированных пользователей", diff --git a/packages/rocketchat-i18n/i18n/sq.i18n.json b/packages/rocketchat-i18n/i18n/sq.i18n.json index 2789b1173d7f..0da84a902bdd 100644 --- a/packages/rocketchat-i18n/i18n/sq.i18n.json +++ b/packages/rocketchat-i18n/i18n/sq.i18n.json @@ -21,8 +21,6 @@ "Accounts_AllowUserProfileChange": "Lejo përdoruesit Profilin Change", "Accounts_AvatarResize": "Ndrysho madhësin e Avatarit", "Accounts_AvatarSize": "Madhësia e Avatarit", - "Accounts_AvatarStorePath": "Avatari Storage Path", - "Accounts_AvatarStoreType": "Avatari Storage Lloji", "Accounts_BlockedDomainsList": "Blocked Lista Domains", "Accounts_BlockedDomainsList_Description": "lista të ndara me presje të fushave të bllokuara", "Accounts_BlockedUsernameList": "Blocked List Emri i përdoruesit", diff --git a/packages/rocketchat-i18n/i18n/sr.i18n.json b/packages/rocketchat-i18n/i18n/sr.i18n.json index 86b6a9570fcb..7b658902e6de 100644 --- a/packages/rocketchat-i18n/i18n/sr.i18n.json +++ b/packages/rocketchat-i18n/i18n/sr.i18n.json @@ -21,8 +21,6 @@ "Accounts_AllowUserProfileChange": "Дозволите Усер Профиле Цханге", "Accounts_AvatarResize": "ресизе Аватари", "Accounts_AvatarSize": "аватар величина", - "Accounts_AvatarStorePath": "Аватар складиштења Пут", - "Accounts_AvatarStoreType": "Аватар Стораге Типе:", "Accounts_BlockedDomainsList": "Блокиран доменов", "Accounts_BlockedDomainsList_Description": "Зарезом одвојена листа блокираних домена", "Accounts_BlockedUsernameList": "Блокиран име Списак", diff --git a/packages/rocketchat-i18n/i18n/sv.i18n.json b/packages/rocketchat-i18n/i18n/sv.i18n.json index b677c0687450..90899745d5b9 100644 --- a/packages/rocketchat-i18n/i18n/sv.i18n.json +++ b/packages/rocketchat-i18n/i18n/sv.i18n.json @@ -27,8 +27,6 @@ "Accounts_AllowUserProfileChange": "Tillåt byte av användarprofil", "Accounts_AvatarResize": "Ändra storlek på avatarer", "Accounts_AvatarSize": "Storlek på avatar", - "Accounts_AvatarStorePath": "Sökväg till avatarförråd", - "Accounts_AvatarStoreType": "Typ av avatarförråd", "Accounts_BlockedDomainsList": "Lista över blockerade domäner", "Accounts_BlockedDomainsList_Description": "Kommaseparerad lista över blockerade domäner", "Accounts_BlockedUsernameList": "Användarlista över blockerade", diff --git a/packages/rocketchat-i18n/i18n/ta-IN.i18n.json b/packages/rocketchat-i18n/i18n/ta-IN.i18n.json index 58ea59cd2b62..9d857fdf3254 100644 --- a/packages/rocketchat-i18n/i18n/ta-IN.i18n.json +++ b/packages/rocketchat-i18n/i18n/ta-IN.i18n.json @@ -21,8 +21,6 @@ "Accounts_AllowUserProfileChange": "பயனர் விவரம் மாற்றம் அனுமதி", "Accounts_AvatarResize": "அவதாரங்களை அளவை", "Accounts_AvatarSize": "avatar அளவு", - "Accounts_AvatarStorePath": "அவதார் சேமிப்பு பாதை", - "Accounts_AvatarStoreType": "அவதார் சேமிப்பு வகை", "Accounts_BlockedDomainsList": "தடுக்கப்பட்ட களங்கள் பட்டியல்", "Accounts_BlockedDomainsList_Description": "தடுக்கப்பட்டது களங்களின் கமாவால் பிரிக்கப்பட்ட பட்டியல்", "Accounts_BlockedUsernameList": "தடுக்கப்பட்ட பயனர் பெயர் பட்டியல்", diff --git a/packages/rocketchat-i18n/i18n/tr.i18n.json b/packages/rocketchat-i18n/i18n/tr.i18n.json index 1f577efdd77d..c70b29e1ba59 100644 --- a/packages/rocketchat-i18n/i18n/tr.i18n.json +++ b/packages/rocketchat-i18n/i18n/tr.i18n.json @@ -27,8 +27,6 @@ "Accounts_AllowUserProfileChange": "Kullanıcının Profilini Değiştirmesine izin ver", "Accounts_AvatarResize": "Profil Resimlerini yeniden boyutlandır", "Accounts_AvatarSize": "Profil Resmi Boyutu", - "Accounts_AvatarStorePath": "Profil Resmi Depo Yolu", - "Accounts_AvatarStoreType": "Profil Resmi Depolama Türü", "Accounts_BlockedDomainsList": "Engellenen Alanlar Listesi", "Accounts_BlockedDomainsList_Description": "Engellenen alanların virgülle ayrılmış listesi", "Accounts_BlockedUsernameList": "Engellenen Kullanıcı Adı Listesi", diff --git a/packages/rocketchat-i18n/i18n/ug.i18n.json b/packages/rocketchat-i18n/i18n/ug.i18n.json index d3a1dc7c7cfa..ba53f4000c38 100644 --- a/packages/rocketchat-i18n/i18n/ug.i18n.json +++ b/packages/rocketchat-i18n/i18n/ug.i18n.json @@ -21,8 +21,6 @@ "Accounts_AllowUserProfileChange": "ئاكونت ئەزانىڭ ماتېرىيالىنى ئۆزگەرتىشكە رۇخسەت قىلدى", "Accounts_AvatarResize": "ئاكونت باش سۈرئەتنىڭ چوڭ كىچىكلىكىنى تەڭشەش", "Accounts_AvatarSize": "ئاكونت باش سۈرئەتنىڭ چوڭ كىچىكلىكى", - "Accounts_AvatarStorePath": "ئاكونت باش سۈرئەت ساقلاش ئادرېسى", - "Accounts_AvatarStoreType": "ئاكونت باش سۈرئەت ساقلاش تىپى", "Accounts_BlockedDomainsList": "ئاكونت تور بەت نامى تىزىملىكىنى توسۇۋېتىلدى", "Accounts_BlockedDomainsList_Description": "چىكىتلىك پەش ئارقىلىق توسۇۋېتىلگەن توربەت نامى تىزىملىكى", "Accounts_BlockedUsernameList": "ئاكونت مەنئىي قىلغان ئەزا تىزىملىكى", diff --git a/packages/rocketchat-i18n/i18n/uk.i18n.json b/packages/rocketchat-i18n/i18n/uk.i18n.json index ff1c6365af7e..9cdb267deb27 100644 --- a/packages/rocketchat-i18n/i18n/uk.i18n.json +++ b/packages/rocketchat-i18n/i18n/uk.i18n.json @@ -25,8 +25,6 @@ "Accounts_AllowUserProfileChange": "Дозволити користувачеві змінювати налаштування профілю", "Accounts_AvatarResize": "Змінити розмір аватару", "Accounts_AvatarSize": "Розмір аватару", - "Accounts_AvatarStorePath": "Шлях до сховища з аватарами", - "Accounts_AvatarStoreType": "Тип сховища с аватарами", "Accounts_BlockedDomainsList": "Перелік заблокованих доменів", "Accounts_BlockedDomainsList_Description": "Перелік заблокованих доменів, розділених комою", "Accounts_BlockedUsernameList": "Перелік заблокованих користувачів", diff --git a/packages/rocketchat-i18n/i18n/zh-HK.i18n.json b/packages/rocketchat-i18n/i18n/zh-HK.i18n.json index 549746827755..e3365a6452c6 100644 --- a/packages/rocketchat-i18n/i18n/zh-HK.i18n.json +++ b/packages/rocketchat-i18n/i18n/zh-HK.i18n.json @@ -20,7 +20,6 @@ "Accounts_AllowUserProfileChange": "允許使用者變更個人檔案", "Accounts_AvatarResize": "調整頭像大\b小", "Accounts_AvatarSize": "頭像大\b小", - "Accounts_AvatarStorePath": "頭像儲存路徑", "Accounts_BlockedDomainsList": "黑名單網域列表", "Accounts_denyUnverifiedEmail": "拒绝未经验证的电子邮件", "Accounts_EmailVerification": "邮件验证", diff --git a/packages/rocketchat-i18n/i18n/zh-TW.i18n.json b/packages/rocketchat-i18n/i18n/zh-TW.i18n.json index d06f9788e46e..53c8db1dc545 100644 --- a/packages/rocketchat-i18n/i18n/zh-TW.i18n.json +++ b/packages/rocketchat-i18n/i18n/zh-TW.i18n.json @@ -25,8 +25,6 @@ "Accounts_AllowUserProfileChange": "允許變更使用者個人檔案", "Accounts_AvatarResize": "調整大頭貼尺寸", "Accounts_AvatarSize": "大頭貼尺寸", - "Accounts_AvatarStorePath": "大頭貼儲存路徑", - "Accounts_AvatarStoreType": "大頭貼儲存類型", "Accounts_BlockedDomainsList": "黑名單網域列表", "Accounts_BlockedDomainsList_Description": "以逗號分隔的網域黑名單", "Accounts_BlockedUsernameList": "使用者黑名單", diff --git a/packages/rocketchat-i18n/i18n/zh.i18n.json b/packages/rocketchat-i18n/i18n/zh.i18n.json index 00b01cda799a..4f1c6f29eb57 100644 --- a/packages/rocketchat-i18n/i18n/zh.i18n.json +++ b/packages/rocketchat-i18n/i18n/zh.i18n.json @@ -27,8 +27,6 @@ "Accounts_AllowUserProfileChange": "允许修改个人资料", "Accounts_AvatarResize": "调整头像大小", "Accounts_AvatarSize": "头像大小", - "Accounts_AvatarStorePath": "头像存储路径", - "Accounts_AvatarStoreType": "头像存储类型", "Accounts_BlockedDomainsList": "已屏蔽域名列表", "Accounts_BlockedDomainsList_Description": "以逗号分隔的屏蔽的域名列表", "Accounts_BlockedUsernameList": "已屏蔽的用户名列表", diff --git a/packages/rocketchat-lib/server/startup/settings.js b/packages/rocketchat-lib/server/startup/settings.js index ae4024737caa..2bfcd3c2b068 100644 --- a/packages/rocketchat-lib/server/startup/settings.js +++ b/packages/rocketchat-lib/server/startup/settings.js @@ -169,25 +169,7 @@ RocketChat.settings.addGroup('Accounts', function() { value: true } }); - this.add('Accounts_AvatarStoreType', 'GridFS', { - type: 'select', - values: [ - { - key: 'GridFS', - i18nLabel: 'GridFS' - }, { - key: 'FileSystem', - i18nLabel: 'FileSystem' - } - ] - }); - this.add('Accounts_AvatarStorePath', '', { - type: 'string', - enableQuery: { - _id: 'Accounts_AvatarStoreType', - value: 'FileSystem' - } - }); + return this.add('Accounts_SetDefaultAvatar', true, { type: 'boolean' }); From a1fed82e0e7f4c444577cb6197528d8a68a30070 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Mon, 29 May 2017 19:13:21 -0300 Subject: [PATCH 36/42] Uploads: Add avatar migration --- .../server/lib/FileUpload.js | 2 +- server/startup/migrations/v096.js | 87 +++++++++++-------- 2 files changed, 54 insertions(+), 35 deletions(-) diff --git a/packages/rocketchat-file-upload/server/lib/FileUpload.js b/packages/rocketchat-file-upload/server/lib/FileUpload.js index cca5a0d60105..748e8fa7336d 100644 --- a/packages/rocketchat-file-upload/server/lib/FileUpload.js +++ b/packages/rocketchat-file-upload/server/lib/FileUpload.js @@ -272,7 +272,7 @@ export class FileUploadClass { try { if (streamOrBuffer instanceof stream) { - stream.pipe(fs.createWriteStream(tmpFile)); + streamOrBuffer.pipe(fs.createWriteStream(tmpFile)); } else if (streamOrBuffer instanceof Buffer) { fs.writeFileSync(tmpFile, streamOrBuffer); } else { diff --git a/server/startup/migrations/v096.js b/server/startup/migrations/v096.js index 60d838169c7a..d896ce0c3a7d 100644 --- a/server/startup/migrations/v096.js +++ b/server/startup/migrations/v096.js @@ -75,49 +75,68 @@ RocketChat.Migrations.add({ 'linkedin' ]; - const avatarsFileStore = FileUpload.getStore('Avatars'); + Meteor.setTimeout(function() { + Meteor.runAsUser('rocket.cat', function() { + const avatarsFileStore = FileUpload.getStore('Avatars'); - const oldAvatarGridFS = new RocketChatFile.GridFS({ - name: 'avatars' - }); - - - RocketChat.models.Users.find({avatarOrigin: {$in: avatarOrigins}}, {avatarOrigin: 1, username: 1}).forEach((user) => { - const id = `${ user.username }.jpg`; - const gridFSAvatar = oldAvatarGridFS.getFileWithReadStream(id); + const oldAvatarGridFS = new RocketChatFile.GridFS({ + name: 'avatars' + }); - if (gridFSAvatar) { - const details = { - userId: user._id, - type: gridFSAvatar.contentType, - size: gridFSAvatar.length - }; + RocketChat.models.Users.find({avatarOrigin: {$in: avatarOrigins}}, {avatarOrigin: 1, username: 1}).forEach((user) => { + const id = `${ user.username }.jpg`; + const gridFSAvatar = oldAvatarGridFS.getFileWithReadStream(id); - avatarsFileStore.insert(details, gridFSAvatar.readStream, () => { - oldAvatarGridFS.deleteFile(id); - }); - } else { - const avatarsPath = RocketChat.settings.get('Accounts_AvatarStorePath'); - if (avatarsPath && avatarsPath.trim()) { - const filePath = path.join(avatarsPath, id); - const stat = fs.statSync(filePath); - if (stat && stat.isFile()) { - const rs = fs.createReadStream(filePath); + if (gridFSAvatar) { const details = { userId: user._id, - type: 'image/jpeg', - size: stat.size + type: gridFSAvatar.contentType, + size: gridFSAvatar.length, + name: user.username }; - avatarsFileStore.insert(details, rs, () => { - fs.unlinkSync(filePath); + avatarsFileStore.insert(details, gridFSAvatar.readStream, (err) => { + if (err) { + console.log({err}); + } else { + oldAvatarGridFS.deleteFile(id); + } }); + } else { + const avatarsPath = RocketChat.models.Settings.findOne({_id: 'Accounts_AvatarStorePath'}).value; + if (avatarsPath && avatarsPath.trim()) { + const filePath = path.join(avatarsPath, id); + try { + const stat = fs.statSync(filePath); + if (stat && stat.isFile()) { + const rs = fs.createReadStream(filePath); + const details = { + userId: user._id, + type: 'image/jpeg', + size: stat.size, + name: user.username + }; + + avatarsFileStore.insert(details, rs, (err) => { + if (err) { + console.log({err}); + } else { + fs.unlinkSync(filePath); + } + }); + } + } catch (e) { + console.log('Error migrating old avatars', e); + } + } } - } - } - }); + }); + + RocketChat.models.Settings.remove({_id: 'Accounts_AvatarStoreType'}); + RocketChat.models.Settings.remove({_id: 'Accounts_AvatarStorePath'}); - RocketChat.models.Settings.remove({_id: 'Accounts_AvatarStoreType'}); - RocketChat.models.Settings.remove({_id: 'Accounts_AvatarStorePath'}); + // TODO: Remove old collections + }); + }, 1000); } }); From 69a76ceb49a8a4a7739f6b35e359a31a422c8b5b Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Tue, 30 May 2017 09:55:43 -0300 Subject: [PATCH 37/42] Uploads: Better file organization --- .../globalFileRestrictions.js | 2 - packages/rocketchat-file-upload/package.js | 8 +- ...onfigFileUploadAmazonS3.js => AmazonS3.js} | 0 ...gFileUploadFileSystem.js => FileSystem.js} | 0 ...ploadGoogleStorage.js => GoogleStorage.js} | 0 .../{configFileUploadGridFS.js => GridFS.js} | 0 .../server/config/Slingshot_DEPRECATED.js | 114 ++++++++++++++++++ ...configStore.js => _configUploadStorage.js} | 6 + .../configFileUploadAmazonS3_Deprecated.js | 79 ------------ ...onfigFileUploadGoogleStorage_Deprecated.js | 56 --------- 10 files changed, 121 insertions(+), 144 deletions(-) rename packages/rocketchat-file-upload/server/config/{configFileUploadAmazonS3.js => AmazonS3.js} (100%) rename packages/rocketchat-file-upload/server/config/{configFileUploadFileSystem.js => FileSystem.js} (100%) rename packages/rocketchat-file-upload/server/config/{configFileUploadGoogleStorage.js => GoogleStorage.js} (100%) rename packages/rocketchat-file-upload/server/config/{configFileUploadGridFS.js => GridFS.js} (100%) create mode 100644 packages/rocketchat-file-upload/server/config/Slingshot_DEPRECATED.js rename packages/rocketchat-file-upload/server/config/{configStore.js => _configUploadStorage.js} (75%) delete mode 100644 packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3_Deprecated.js delete mode 100644 packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage_Deprecated.js diff --git a/packages/rocketchat-file-upload/globalFileRestrictions.js b/packages/rocketchat-file-upload/globalFileRestrictions.js index 8e2d5086dee2..0568c018d8f1 100644 --- a/packages/rocketchat-file-upload/globalFileRestrictions.js +++ b/packages/rocketchat-file-upload/globalFileRestrictions.js @@ -25,7 +25,5 @@ const slingShotConfig = { allowedFileTypes: null }; -Slingshot.fileRestrictions('rocketchat-avatars', slingShotConfig); Slingshot.fileRestrictions('rocketchat-uploads', slingShotConfig); Slingshot.fileRestrictions('rocketchat-uploads-gs', slingShotConfig); -Slingshot.fileRestrictions('rocketchat-avatars-gs', slingShotConfig); diff --git a/packages/rocketchat-file-upload/package.js b/packages/rocketchat-file-upload/package.js index 8a22b09859a4..85e698199a4a 100644 --- a/packages/rocketchat-file-upload/package.js +++ b/packages/rocketchat-file-upload/package.js @@ -33,13 +33,7 @@ Package.onUse(function(api) { api.addFiles('server/lib/FileUpload.js', 'server'); api.addFiles('server/lib/requests.js', 'server'); - api.addFiles('server/config/configFileUploadAmazonS3.js', 'server'); - api.addFiles('server/config/configFileUploadAmazonS3_Deprecated.js', 'server'); - api.addFiles('server/config/configFileUploadFileSystem.js', 'server'); - api.addFiles('server/config/configFileUploadGoogleStorage.js', 'server'); - api.addFiles('server/config/configFileUploadGoogleStorage_Deprecated.js', 'server'); - api.addFiles('server/config/configFileUploadGridFS.js', 'server'); - api.addFiles('server/config/configStore.js', 'server'); + api.addFiles('server/config/_configUploadStorage.js', 'server'); api.addFiles('server/methods/sendFileMessage.js', 'server'); api.addFiles('server/methods/getS3FileUrl.js', 'server'); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js b/packages/rocketchat-file-upload/server/config/AmazonS3.js similarity index 100% rename from packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3.js rename to packages/rocketchat-file-upload/server/config/AmazonS3.js diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js b/packages/rocketchat-file-upload/server/config/FileSystem.js similarity index 100% rename from packages/rocketchat-file-upload/server/config/configFileUploadFileSystem.js rename to packages/rocketchat-file-upload/server/config/FileSystem.js diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js b/packages/rocketchat-file-upload/server/config/GoogleStorage.js similarity index 100% rename from packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage.js rename to packages/rocketchat-file-upload/server/config/GoogleStorage.js diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js b/packages/rocketchat-file-upload/server/config/GridFS.js similarity index 100% rename from packages/rocketchat-file-upload/server/config/configFileUploadGridFS.js rename to packages/rocketchat-file-upload/server/config/GridFS.js diff --git a/packages/rocketchat-file-upload/server/config/Slingshot_DEPRECATED.js b/packages/rocketchat-file-upload/server/config/Slingshot_DEPRECATED.js new file mode 100644 index 000000000000..45d245a4d3a4 --- /dev/null +++ b/packages/rocketchat-file-upload/server/config/Slingshot_DEPRECATED.js @@ -0,0 +1,114 @@ +/* globals Slingshot, FileUpload */ + +const configureSlingshot = _.debounce(() => { + const type = RocketChat.settings.get('FileUpload_Storage_Type'); + const bucket = RocketChat.settings.get('FileUpload_S3_Bucket'); + const acl = RocketChat.settings.get('FileUpload_S3_Acl'); + const accessKey = RocketChat.settings.get('FileUpload_S3_AWSAccessKeyId'); + const secretKey = RocketChat.settings.get('FileUpload_S3_AWSSecretAccessKey'); + const cdn = RocketChat.settings.get('FileUpload_S3_CDN'); + const region = RocketChat.settings.get('FileUpload_S3_Region'); + const bucketUrl = RocketChat.settings.get('FileUpload_S3_BucketURL'); + + delete Slingshot._directives['rocketchat-uploads']; + + if (type === 'AmazonS3' && !_.isEmpty(bucket) && !_.isEmpty(accessKey) && !_.isEmpty(secretKey)) { + if (Slingshot._directives['rocketchat-uploads']) { + delete Slingshot._directives['rocketchat-uploads']; + } + const config = { + bucket, + key(file, metaContext) { + const id = Random.id(); + const path = `${ RocketChat.settings.get('uniqueID') }/uploads/${ metaContext.rid }/${ this.userId }/${ id }`; + + const upload = { + _id: id, + rid: metaContext.rid, + AmazonS3: { + path + } + }; + + RocketChat.models.Uploads.insertFileInit(this.userId, 'AmazonS3:Uploads', file, upload); + + return path; + }, + AWSAccessKeyId: accessKey, + AWSSecretAccessKey: secretKey + }; + + if (!_.isEmpty(acl)) { + config.acl = acl; + } + + if (!_.isEmpty(cdn)) { + config.cdn = cdn; + } + + if (!_.isEmpty(region)) { + config.region = region; + } + + if (!_.isEmpty(bucketUrl)) { + config.bucketUrl = bucketUrl; + } + + try { + Slingshot.createDirective('rocketchat-uploads', Slingshot.S3Storage, config); + } catch (e) { + console.error('Error configuring S3 ->', e.message); + } + } +}, 500); + +RocketChat.settings.get('FileUpload_Storage_Type', configureSlingshot); +RocketChat.settings.get(/^FileUpload_S3_/, configureSlingshot); + + + +const createGoogleStorageDirective = _.debounce(() => { + const type = RocketChat.settings.get('FileUpload_Storage_Type'); + const bucket = RocketChat.settings.get('FileUpload_GoogleStorage_Bucket'); + const accessId = RocketChat.settings.get('FileUpload_GoogleStorage_AccessId'); + const secret = RocketChat.settings.get('FileUpload_GoogleStorage_Secret'); + + delete Slingshot._directives['rocketchat-uploads-gs']; + + if (type === 'GoogleCloudStorage' && !_.isEmpty(secret) && !_.isEmpty(accessId) && !_.isEmpty(bucket)) { + if (Slingshot._directives['rocketchat-uploads-gs']) { + delete Slingshot._directives['rocketchat-uploads-gs']; + } + + const config = { + bucket, + GoogleAccessId: accessId, + GoogleSecretKey: secret, + key(file, metaContext) { + const id = Random.id(); + const path = `${ RocketChat.settings.get('uniqueID') }/uploads/${ metaContext.rid }/${ this.userId }/${ id }`; + + const upload = { + _id: id, + rid: metaContext.rid, + GoogleStorage: { + path + } + }; + + RocketChat.models.Uploads.insertFileInit(this.userId, 'GoogleCloudStorage:Uploads', file, upload); + + return path; + } + }; + + try { + Slingshot.createDirective('rocketchat-uploads-gs', Slingshot.GoogleCloud, config); + } catch (e) { + console.error('Error configuring GoogleCloudStorage ->', e.message); + } + } +}, 500); + +RocketChat.settings.get('FileUpload_Storage_Type', createGoogleStorageDirective); +RocketChat.settings.get(/^FileUpload_GoogleStorage_/, createGoogleStorageDirective); diff --git a/packages/rocketchat-file-upload/server/config/configStore.js b/packages/rocketchat-file-upload/server/config/_configUploadStorage.js similarity index 75% rename from packages/rocketchat-file-upload/server/config/configStore.js rename to packages/rocketchat-file-upload/server/config/_configUploadStorage.js index 1fc3e1bdbb98..df56dc4d307a 100644 --- a/packages/rocketchat-file-upload/server/config/configStore.js +++ b/packages/rocketchat-file-upload/server/config/_configUploadStorage.js @@ -1,5 +1,11 @@ /* globals UploadFS */ +import './AmazonS3.js'; +import './FileSystem.js'; +import './GoogleStorage.js'; +import './GridFS.js'; +import './Slingshot_DEPRECATED.js'; + const configStore = _.debounce(() => { const store = RocketChat.settings.get('FileUpload_Storage_Type'); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3_Deprecated.js b/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3_Deprecated.js deleted file mode 100644 index 958bc87cbc57..000000000000 --- a/packages/rocketchat-file-upload/server/config/configFileUploadAmazonS3_Deprecated.js +++ /dev/null @@ -1,79 +0,0 @@ -/* globals Slingshot, FileUpload */ - -function createDirective(directiveName, { key, bucket, accessKey, secretKey, region, acl, cdn, bucketUrl}) { - if (Slingshot._directives[directiveName]) { - delete Slingshot._directives[directiveName]; - } - const config = { - bucket, - key, - AWSAccessKeyId: accessKey, - AWSSecretAccessKey: secretKey - }; - - if (!_.isEmpty(acl)) { - config.acl = acl; - } - - if (!_.isEmpty(cdn)) { - config.cdn = cdn; - } - - if (!_.isEmpty(region)) { - config.region = region; - } - - if (!_.isEmpty(bucketUrl)) { - config.bucketUrl = bucketUrl; - } - - try { - Slingshot.createDirective(directiveName, Slingshot.S3Storage, config); - } catch (e) { - console.error('Error configuring S3 ->', e.message); - } -} - -const configureSlingshot = _.debounce(() => { - const type = RocketChat.settings.get('FileUpload_Storage_Type'); - const bucket = RocketChat.settings.get('FileUpload_S3_Bucket'); - const acl = RocketChat.settings.get('FileUpload_S3_Acl'); - const accessKey = RocketChat.settings.get('FileUpload_S3_AWSAccessKeyId'); - const secretKey = RocketChat.settings.get('FileUpload_S3_AWSSecretAccessKey'); - const cdn = RocketChat.settings.get('FileUpload_S3_CDN'); - const region = RocketChat.settings.get('FileUpload_S3_Region'); - const bucketUrl = RocketChat.settings.get('FileUpload_S3_BucketURL'); - - delete Slingshot._directives['rocketchat-uploads']; - - if (type === 'AmazonS3' && !_.isEmpty(bucket) && !_.isEmpty(accessKey) && !_.isEmpty(secretKey)) { - createDirective('rocketchat-uploads', { - key(file, metaContext) { - const id = Random.id(); - const path = `${ RocketChat.settings.get('uniqueID') }/uploads/${ metaContext.rid }/${ this.userId }/${ id }`; - - const upload = { - _id: id, - rid: metaContext.rid, - AmazonS3: { - path - } - }; - - RocketChat.models.Uploads.insertFileInit(this.userId, 'AmazonS3:Uploads', file, upload); - - return path; - }, - bucket, - accessKey, - secretKey, - region, - acl, - cdn, - bucketUrl - }); - } -}, 500); - -RocketChat.settings.get('FileUpload_Storage_Type', configureSlingshot); -RocketChat.settings.get(/^FileUpload_S3_/, configureSlingshot); diff --git a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage_Deprecated.js b/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage_Deprecated.js deleted file mode 100644 index c4fcda5ecf08..000000000000 --- a/packages/rocketchat-file-upload/server/config/configFileUploadGoogleStorage_Deprecated.js +++ /dev/null @@ -1,56 +0,0 @@ -/* globals FileUpload, Slingshot */ - -function createDirective(directiveName, { key, bucket, accessId, secret }) { - if (Slingshot._directives[directiveName]) { - delete Slingshot._directives[directiveName]; - } - - const config = { - bucket, - GoogleAccessId: accessId, - GoogleSecretKey: secret, - key - }; - - try { - Slingshot.createDirective(directiveName, Slingshot.GoogleCloud, config); - } catch (e) { - console.error('Error configuring GoogleCloudStorage ->', e.message); - } -} - -const createGoogleStorageDirective = _.debounce(() => { - const type = RocketChat.settings.get('FileUpload_Storage_Type'); - const bucket = RocketChat.settings.get('FileUpload_GoogleStorage_Bucket'); - const accessId = RocketChat.settings.get('FileUpload_GoogleStorage_AccessId'); - const secret = RocketChat.settings.get('FileUpload_GoogleStorage_Secret'); - - delete Slingshot._directives['rocketchat-uploads-gs']; - - if (type === 'GoogleCloudStorage' && !_.isEmpty(secret) && !_.isEmpty(accessId) && !_.isEmpty(bucket)) { - createDirective('rocketchat-uploads-gs', { - key: function _googleCloudStorageKey(file, metaContext) { - const id = Random.id(); - const path = `${ RocketChat.settings.get('uniqueID') }/uploads/${ metaContext.rid }/${ this.userId }/${ id }`; - - const upload = { - _id: id, - rid: metaContext.rid, - GoogleStorage: { - path - } - }; - - RocketChat.models.Uploads.insertFileInit(this.userId, 'GoogleCloudStorage:Uploads', file, upload); - - return path; - }, - bucket, - accessId, - secret - }); - } -}, 500); - -RocketChat.settings.get('FileUpload_Storage_Type', createGoogleStorageDirective); -RocketChat.settings.get(/^FileUpload_GoogleStorage_/, createGoogleStorageDirective); From e14e73d08703ada05070cd076e82912dd461bb65 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Tue, 30 May 2017 10:05:34 -0300 Subject: [PATCH 38/42] Uploads: Implement range for google storage read --- .../ufs/GoogleStorage/server.js | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/rocketchat-file-upload/ufs/GoogleStorage/server.js b/packages/rocketchat-file-upload/ufs/GoogleStorage/server.js index a6536d0a3342..8a764caac732 100644 --- a/packages/rocketchat-file-upload/ufs/GoogleStorage/server.js +++ b/packages/rocketchat-file-upload/ufs/GoogleStorage/server.js @@ -83,9 +83,18 @@ export class GoogleStorageStore extends UploadFS.Store { * @param options * @return {*} */ - this.getReadStream = function(fileId, file/*, options = {}*/) { - // TODO range? - return this.bucket.file(this.getPath(file)).createReadStream(); + this.getReadStream = function(fileId, file, options = {}) { + const config = {}; + + if (options.start != null) { + config.start = options.start; + } + + if (options.end != null) { + config.end = options.end; + } + + return this.bucket.file(this.getPath(file)).createReadStream(config); }; /** From 4d94d13ed17ef5b4497d11bff60920f8c96943c9 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Tue, 30 May 2017 10:12:37 -0300 Subject: [PATCH 39/42] Uploads: Finish migration --- server/startup/migrations/v096.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/startup/migrations/v096.js b/server/startup/migrations/v096.js index d896ce0c3a7d..b7069fea8e56 100644 --- a/server/startup/migrations/v096.js +++ b/server/startup/migrations/v096.js @@ -135,7 +135,10 @@ RocketChat.Migrations.add({ RocketChat.models.Settings.remove({_id: 'Accounts_AvatarStoreType'}); RocketChat.models.Settings.remove({_id: 'Accounts_AvatarStorePath'}); - // TODO: Remove old collections + const avatarsFiles = new Mongo.Collection('avatars.files'); + const avatarsChunks = new Mongo.Collection('avatars.chunks'); + avatarsFiles.rawCollection().drop(); + avatarsChunks.rawCollection().drop(); }); }, 1000); } From 3586b26fef4401452f9890c19cad65d28b854c17 Mon Sep 17 00:00:00 2001 From: Diego Sampaio Date: Wed, 31 May 2017 09:44:08 -0300 Subject: [PATCH 40/42] Fix avatar migration removing new avatar entries --- server/startup/migrations/v096.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/startup/migrations/v096.js b/server/startup/migrations/v096.js index b7069fea8e56..a6554a37575f 100644 --- a/server/startup/migrations/v096.js +++ b/server/startup/migrations/v096.js @@ -113,8 +113,7 @@ RocketChat.Migrations.add({ const details = { userId: user._id, type: 'image/jpeg', - size: stat.size, - name: user.username + size: stat.size }; avatarsFileStore.insert(details, rs, (err) => { From 545eeac7ecf1b828f055b25fd17702e9c994e230 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Wed, 31 May 2017 17:53:27 -0300 Subject: [PATCH 41/42] Fix tests --- packages/rocketchat-api/server/api.js | 2 +- packages/rocketchat-file-upload/server/config/AmazonS3.js | 4 ++++ .../rocketchat-file-upload/server/config/GoogleStorage.js | 4 ++++ packages/rocketchat-lib/server/functions/setUsername.js | 4 +++- 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/packages/rocketchat-api/server/api.js b/packages/rocketchat-api/server/api.js index f1e71eeaa5a2..a8ee3da879d6 100644 --- a/packages/rocketchat-api/server/api.js +++ b/packages/rocketchat-api/server/api.js @@ -104,7 +104,7 @@ class API extends Restivus { try { result = originalAction.apply(this); } catch (e) { - this.logger.debug(`${ method } ${ route } threw an error:`, e); + this.logger.debug(`${ method } ${ route } threw an error:`, e.stack); return RocketChat.API.v1.failure(e.message, e.error); } diff --git a/packages/rocketchat-file-upload/server/config/AmazonS3.js b/packages/rocketchat-file-upload/server/config/AmazonS3.js index 0f98c4a9e7d3..569e1460c949 100644 --- a/packages/rocketchat-file-upload/server/config/AmazonS3.js +++ b/packages/rocketchat-file-upload/server/config/AmazonS3.js @@ -35,6 +35,10 @@ const configure = _.debounce(function() { // const CDN = RocketChat.settings.get('FileUpload_S3_CDN'); // const BucketURL = RocketChat.settings.get('FileUpload_S3_BucketURL'); + if (!Bucket || !AWSAccessKeyId || !AWSSecretAccessKey) { + return; + } + const config = { connection: { accessKeyId: AWSAccessKeyId, diff --git a/packages/rocketchat-file-upload/server/config/GoogleStorage.js b/packages/rocketchat-file-upload/server/config/GoogleStorage.js index 67fcc6b75c34..2010cfa68b99 100644 --- a/packages/rocketchat-file-upload/server/config/GoogleStorage.js +++ b/packages/rocketchat-file-upload/server/config/GoogleStorage.js @@ -36,6 +36,10 @@ const configure = _.debounce(function() { const secret = RocketChat.settings.get('FileUpload_GoogleStorage_Secret'); const URLExpiryTimeSpan = RocketChat.settings.get('FileUpload_S3_URLExpiryTimeSpan'); + if (!bucket || !accessId || !secret) { + return; + } + const config = { connection: { credentials: { diff --git a/packages/rocketchat-lib/server/functions/setUsername.js b/packages/rocketchat-lib/server/functions/setUsername.js index 013361ebd798..20eeabdccf0f 100644 --- a/packages/rocketchat-lib/server/functions/setUsername.js +++ b/packages/rocketchat-lib/server/functions/setUsername.js @@ -68,7 +68,9 @@ RocketChat._setUsername = function(userId, u) { const fileStore = FileUpload.getStore('Avatars'); const file = fileStore.model.findOneByName(previousUsername); - fileStore.model.updateFileNameById(file._id, username); + if (file) { + fileStore.model.updateFileNameById(file._id, username); + } } // Set new username* RocketChat.models.Users.setUsername(user._id, username); From b827c575214aa23dd6a2406cc086e643e7523428 Mon Sep 17 00:00:00 2001 From: Rodrigo Nascimento Date: Wed, 31 May 2017 19:05:37 -0300 Subject: [PATCH 42/42] Unit: Fix mentions tests --- packages/rocketchat-mentions/Mentions.js | 12 ++- packages/rocketchat-mentions/client.js | 3 + .../rocketchat-mentions/tests/client.tests.js | 80 +++++++++++++++++-- 3 files changed, 88 insertions(+), 7 deletions(-) diff --git a/packages/rocketchat-mentions/Mentions.js b/packages/rocketchat-mentions/Mentions.js index b4d8d6317abc..55959a9bed54 100644 --- a/packages/rocketchat-mentions/Mentions.js +++ b/packages/rocketchat-mentions/Mentions.js @@ -4,8 +4,9 @@ */ import _ from 'underscore'; export default class { - constructor({pattern, me}) { + constructor({pattern, useRealName, me}) { this.pattern = pattern; + this.useRealName = useRealName; this.me = me; } set me(m) { @@ -20,6 +21,12 @@ export default class { get pattern() { return typeof this._pattern === 'function' ? this._pattern() : this._pattern; } + set useRealName(s) { + this._useRealName = s; + } + get useRealName() { + return typeof this._useRealName === 'function' ? this._useRealName() : this._useRealName; + } get userMentionRegex() { return new RegExp(`@(${ this.pattern })`, 'gm'); } @@ -36,7 +43,8 @@ export default class { if (message.temp == null && mentionObj == null) { return match; } - const name = RocketChat.settings.get('UI_Use_Real_Name') && mentionObj && mentionObj.name; + const name = this.useRealName && mentionObj && mentionObj.name; + console.log({name}); return `${ name || match }`; }); diff --git a/packages/rocketchat-mentions/client.js b/packages/rocketchat-mentions/client.js index ca2d366ab90a..b95e9bb320e9 100644 --- a/packages/rocketchat-mentions/client.js +++ b/packages/rocketchat-mentions/client.js @@ -3,6 +3,9 @@ const MentionsClient = new Mentions({ pattern() { return RocketChat.settings.get('UTF8_Names_Validation'); }, + useRealName() { + return RocketChat.settings.get('UI_Use_Real_Name'); + }, me() { const me = Meteor.user(); return me && me.username; diff --git a/packages/rocketchat-mentions/tests/client.tests.js b/packages/rocketchat-mentions/tests/client.tests.js index 1c9ef164f627..c6ef427d14f2 100644 --- a/packages/rocketchat-mentions/tests/client.tests.js +++ b/packages/rocketchat-mentions/tests/client.tests.js @@ -25,6 +25,19 @@ describe('Mention', function() { }); }); }); + describe('get useRealName', () => { + beforeEach(() => mention.useRealName = () => true); + describe('by function', function functionName() { + it('should be true', () => { + assert.equal(true, mention.useRealName); + }); + }); + describe('by const', function functionName() { + it('should be true', () => { + assert.equal(true, mention.useRealName); + }); + }); + }); describe('get me', () => { const me = 'me'; describe('by function', function functionName() { @@ -151,7 +164,7 @@ describe('Mention', function() { }); const message = { - mentions:[{username:'rocket.cat'}, {username:'admin'}, {username: 'me'}], + mentions:[{username:'rocket.cat', name: 'Rocket.Cat'}, {username:'admin', name: 'Admin'}, {username: 'me', name: 'Me'}], channels: [{name: 'general'}, {name: 'rocket.cat'}] }; describe('replace methods', function() { @@ -163,12 +176,12 @@ describe('replace methods', function() { const str2 = '@rocket.cat'; it(`shoud render for ${ str2 }`, () => { const result = mention.replaceUsers('@rocket.cat', message, 'me'); - assert.equal(result, `${ str2 }`); + assert.equal(result, `${ str2 }`); }); it(`shoud render for "hello ${ str2 }"`, () => { const result = mention.replaceUsers(`hello ${ str2 }`, message, 'me'); - assert.equal(result, `hello ${ str2 }`); + assert.equal(result, `hello ${ str2 }`); }); it('shoud render for unknow/private user "hello @unknow"', () => { const result = mention.replaceUsers('hello @unknow', message, 'me'); @@ -176,9 +189,40 @@ describe('replace methods', function() { }); it('shoud render for me', () => { const result = mention.replaceUsers('hello @me', message, 'me'); - assert.equal(result, 'hello @me'); + assert.equal(result, 'hello @me'); }); }); + + describe('replaceUsers (RealNames)', () => { + beforeEach(() => { + mention.useRealName = () => true; + }); + it('shoud render for @all', () => { + const result = mention.replaceUsers('@all', message, 'me'); + assert.equal('@all', result); + }); + + const str2 = '@rocket.cat'; + const str2Name = 'Rocket.Cat'; + it(`shoud render for ${ str2 }`, () => { + const result = mention.replaceUsers('@rocket.cat', message, 'me'); + assert.equal(result, `${ str2Name }`); + }); + + it(`shoud render for "hello ${ str2 }"`, () => { + const result = mention.replaceUsers(`hello ${ str2 }`, message, 'me'); + assert.equal(result, `hello ${ str2Name }`); + }); + it('shoud render for unknow/private user "hello @unknow"', () => { + const result = mention.replaceUsers('hello @unknow', message, 'me'); + assert.equal(result, 'hello @unknow'); + }); + it('shoud render for me', () => { + const result = mention.replaceUsers('hello @me', message, 'me'); + assert.equal(result, 'hello Me'); + }); + }); + describe('replaceChannels', () => { it('shoud render for #general', () => { const result = mention.replaceChannels('#general', message, 'me'); @@ -198,6 +242,7 @@ describe('replace methods', function() { assert.equal(result, 'hello #unknow'); }); }); + describe('parse all', () => { it('shoud render for #general', () => { message.html = '#general'; @@ -207,7 +252,7 @@ describe('replace methods', function() { it('shoud render for "#general and @rocket.cat', () => { message.html = '#general and @rocket.cat'; const result = mention.parse(message, 'me'); - assert.equal('#general and @rocket.cat', result.html); + assert.equal('#general and @rocket.cat', result.html); }); it('shoud render for "', () => { message.html = ''; @@ -219,6 +264,31 @@ describe('replace methods', function() { const result = mention.parse(message, 'me'); assert.equal('simple text', result.html); }); + }); + describe('parse all (RealNames)', () => { + beforeEach(() => { + mention.useRealName = () => true; + }); + it('shoud render for #general', () => { + message.html = '#general'; + const result = mention.parse(message, 'me'); + assert.equal('#general', result.html); + }); + it('shoud render for "#general and @rocket.cat', () => { + message.html = '#general and @rocket.cat'; + const result = mention.parse(message, 'me'); + assert.equal('#general and Rocket.Cat', result.html); + }); + it('shoud render for "', () => { + message.html = ''; + const result = mention.parse(message, 'me'); + assert.equal('', result.html); + }); + it('shoud render for "simple text', () => { + message.html = 'simple text'; + const result = mention.parse(message, 'me'); + assert.equal('simple text', result.html); + }); }); });