diff --git a/modules/storage/src/admin/index.ts b/modules/storage/src/admin/index.ts index 2af9d9760..b065d875f 100644 --- a/modules/storage/src/admin/index.ts +++ b/modules/storage/src/admin/index.ts @@ -92,7 +92,7 @@ export class AdminRoutes { isPublic: TYPE.Boolean, }, action: ConduitRouteActions.POST, - path: '/files/uploadByUrl', + path: '/files/upload', description: `Creates a new file and provides a URL to upload it to.`, }, new ConduitRouteReturnDefinition('CreateFileByUrl', File.name), @@ -108,15 +108,34 @@ export class AdminRoutes { }, bodyParams: { name: ConduitString.Optional, - data: ConduitString.Optional, folder: ConduitString.Optional, container: ConduitString.Optional, + data: ConduitString.Required, mimeType: ConduitString.Optional, }, }, new ConduitRouteReturnDefinition('PatchFile', File.name), this.fileHandlers.updateFile.bind(this.fileHandlers), ); + this.routingManager.route( + { + urlParams: { + id: { type: TYPE.String, required: true }, + }, + bodyParams: { + name: ConduitString.Optional, + folder: ConduitString.Optional, + container: ConduitString.Optional, + mimeType: ConduitString.Optional, + size: ConduitNumber.Optional, + }, + action: ConduitRouteActions.PATCH, + path: '/files/upload/:id', + description: `Updates a file and provides a URL to upload its data to.`, + }, + new ConduitRouteReturnDefinition('PatchFileByUrl', 'String'), + this.fileHandlers.updateFileUploadUrl.bind(this.fileHandlers), + ); this.routingManager.route( { path: '/files/:id', diff --git a/modules/storage/src/handlers/file.ts b/modules/storage/src/handlers/file.ts index 61f948607..0542d5835 100644 --- a/modules/storage/src/handlers/file.ts +++ b/modules/storage/src/handlers/file.ts @@ -105,7 +105,7 @@ export class FileHandlers { } try { - return this.getFileUploadUrl( + return await this._createFileUploadUrl( usedContainer, newFolder, isPublic, @@ -121,56 +121,51 @@ export class FileHandlers { } } - async updateFile(call: ParsedRouterRequest): Promise { - const { id, data, name, container, folder, mimeType } = call.request.params; + async updateFileUploadUrl(call: ParsedRouterRequest): Promise { + const { id, mimeType, size } = call.request.params; + const found = await File.getInstance().findOne({ _id: id }); + if (isNil(found)) { + throw new GrpcError(status.NOT_FOUND, 'File does not exist'); + } + const { name, folder, container } = await this.validateFilenameAndContainer( + call, + found, + ); try { - const found = await File.getInstance().findOne({ _id: id }); - if (isNil(found)) { - throw new GrpcError(status.NOT_FOUND, 'File does not exist'); - } - let fileData = await this.storageProvider - .container(found.container) - .get((found.folder ?? '') + found.name); - - if (!isNil(data)) { - fileData = Buffer.from(data, 'base64'); - } - - const newName = name ?? found.name; - let newFolder = folder ?? found.folder; - if (!newFolder.endsWith('/')) { - // existing folder names are currently suffixed by "/" upon creation - newFolder += '/'; - } - const newContainer = container ?? found.container; - found.mimeType = mimeType ?? found.mimeType; - const isDataUpdate = - newName === found.name && - newContainer === found.container && - newFolder === found.folder; - - if (newContainer !== found.container) { - await this.findOrCreateContainer(newContainer); - } - if (newFolder !== found.folder) { - await this.findOrCreateFolder(newFolder, newContainer); - } + return await this._updateFileUploadUrl( + name, + folder, + container, + mimeType ?? found.mimeType, + found, + size, + ); + } catch (e) { + throw new GrpcError( + status.INTERNAL, + (e as Error).message ?? 'Something went wrong', + ); + } + } - const exists = await File.getInstance().findOne({ - name: newName, - container: newContainer, - folder: newFolder, - }); - if (!isDataUpdate && exists) { - throw new GrpcError(status.ALREADY_EXISTS, 'File already exists'); - } - return this.storeUpdatedFile( - newContainer, - newFolder, - newName, + async updateFile(call: ParsedRouterRequest): Promise { + const { id, data, mimeType } = call.request.params; + const found = await File.getInstance().findOne({ _id: id }); + if (isNil(found)) { + throw new GrpcError(status.NOT_FOUND, 'File does not exist'); + } + const { name, folder, container } = await this.validateFilenameAndContainer( + call, + found, + ); + try { + return await this._updateFile( + name, + folder, + container, + Buffer.from(data, 'base64'), + mimeType ?? found.mimeType, found, - fileData, - isDataUpdate, ); } catch (e) { throw new GrpcError( @@ -191,7 +186,7 @@ export class FileHandlers { } const success = await this.storageProvider .container(found.container) - .delete((found.folder ?? '') + found.name); + .delete((found.folder === '/' ? '' : found.folder) + found.name); if (!success) { throw new GrpcError(status.INTERNAL, 'File could not be deleted'); } @@ -218,7 +213,7 @@ export class FileHandlers { } const url = await this.storageProvider .container(found.container) - .getSignedUrl((found.folder ?? '') + found.name); + .getSignedUrl((found.folder === '/' ? '' : found.folder) + found.name); if (!call.request.params.redirect) { return { result: url }; @@ -245,7 +240,9 @@ export class FileHandlers { let data: Buffer; const result = await this.storageProvider .container(file.container) - .get(file.folder ? file.folder + file.name : file.name); + .get( + file.folder ? (file.folder === '/' ? '' : file.folder) + file.name : file.name, + ); if (result instanceof Error) { throw result; } else { @@ -314,14 +311,13 @@ export class FileHandlers { ): Promise { const buffer = Buffer.from(data, 'base64'); const size = buffer.byteLength; - await this.storageProvider .container(container) - .store((folder ?? '') + name, buffer, isPublic); + .store((folder === '/' ? '' : folder) + name, buffer, isPublic); const publicUrl = isPublic ? await this.storageProvider .container(container) - .getPublicUrl((folder ?? '') + name) + .getPublicUrl((folder === '/' ? '' : folder) + name) : null; ConduitGrpcSdk.Metrics?.increment('files_total'); ConduitGrpcSdk.Metrics?.increment('storage_size_bytes_total', size); @@ -336,7 +332,7 @@ export class FileHandlers { }); } - private async getFileUploadUrl( + private async _createFileUploadUrl( container: string, folder: string, isPublic: boolean, @@ -346,11 +342,15 @@ export class FileHandlers { ): Promise { await this.storageProvider .container(container) - .store((folder ?? '') + name, Buffer.from('PENDING UPLOAD'), isPublic); + .store( + (folder === '/' ? '' : folder) + name, + Buffer.from('PENDING UPLOAD'), + isPublic, + ); const publicUrl = isPublic ? await this.storageProvider .container(container) - .getPublicUrl((folder ?? '') + name) + .getPublicUrl((folder === '/' ? '' : folder) + name) : null; ConduitGrpcSdk.Metrics?.increment('files_total'); ConduitGrpcSdk.Metrics?.increment('storage_size_bytes_total', size); @@ -362,43 +362,122 @@ export class FileHandlers { isPublic, url: publicUrl, }); - return (await this.storageProvider .container(container) - .getUploadUrl((folder ?? '') + name)) as string; + .getUploadUrl((folder === '/' ? '' : folder) + name)) as string; } - private async storeUpdatedFile( - container: string, + private async validateFilenameAndContainer(call: ParsedRouterRequest, file: File) { + const { name, folder, container } = call.request.params; + const newName = name ?? file.name; + const newContainer = container ?? file.container; + if (newContainer !== file.container) { + await this.findOrCreateContainer(newContainer); + } + // Existing folder names are currently suffixed by "/" upon creation + const newFolder = isNil(folder) + ? file.folder + : folder.trim().slice(-1) !== '/' + ? folder.trim() + '/' + : folder.trim(); + if (newFolder !== file.folder) { + await this.findOrCreateFolder(newFolder, newContainer); + } + const exists = await File.getInstance().findOne({ + name: newName, + container: newContainer, + folder: newFolder, + }); + if (!isNil(exists)) { + throw new GrpcError(status.ALREADY_EXISTS, 'File already exists'); + } + return { + name: newName, + folder: newFolder, + container: newContainer, + }; + } + + private async _updateFileUploadUrl( + name: string, folder: string, + container: string, + mimeType: string, + file: File, + size: number | undefined | null, + ): Promise { + const onlyDataUpdate = + name === file.name && folder === file.folder && container === file.container; + if (onlyDataUpdate) { + await File.getInstance().findByIdAndUpdate(file._id, { mimeType }); + } else { + await this.storageProvider + .container(container) + .store( + (folder === '/' ? '' : folder) + name, + Buffer.from('PENDING UPLOAD'), + file.isPublic, + ); + await this.storageProvider + .container(file.container) + .delete((file.folder === '/' ? '' : file.folder) + file.name); + const url = file.isPublic + ? await this.storageProvider + .container(container) + .getPublicUrl((folder === '/' ? '' : folder) + name) + : null; + await File.getInstance().findByIdAndUpdate(file._id, { + name, + folder, + container, + url, + mimeType, + }); + } + if (!isNil(size)) this.updateFileMetrics(file.size, size!); + return (await this.storageProvider + .container(container) + .getUploadUrl((folder === '/' ? '' : folder) + name)) as string; + } + + private async _updateFile( name: string, - found: File, - fileData: any, - isDataUpdate: boolean, + folder: string, + container: string, + data: Buffer, + mimeType: string, + file: File, ): Promise { + const onlyDataUpdate = + name === file.name && folder === file.folder && container === file.container; await this.storageProvider .container(container) - .store((folder ?? '') + name, fileData); - // calling delete after store call succeeds - if (!isDataUpdate) { + .store((folder === '/' ? '' : folder) + name, data, file.isPublic); + if (!onlyDataUpdate) { await this.storageProvider - .container(found.container) - .delete((found.folder ?? '') + found.name); + .container(file.container) + .delete((file.folder === '/' ? '' : file.folder) + file.name); } + const url = file.isPublic + ? await this.storageProvider + .container(container) + .getPublicUrl((folder === '/' ? '' : folder) + name) + : null; + const updatedFile = (await File.getInstance().findByIdAndUpdate(file._id, { + name, + folder, + container, + url, + mimeType, + })) as File; + this.updateFileMetrics(file.size, data.byteLength); + return updatedFile; + } - const fileSizeDiff = Math.abs(found.size - fileData.byteLength); + private updateFileMetrics(currentSize: number, newSize: number) { + const fileSizeDiff = Math.abs(currentSize - newSize); fileSizeDiff < 0 ? ConduitGrpcSdk.Metrics?.increment('storage_size_bytes_total', fileSizeDiff) : ConduitGrpcSdk.Metrics?.decrement('storage_size_bytes_total', fileSizeDiff); - if (found.isPublic) { - found.url = await this.storageProvider - .container(container) - .getPublicUrl((folder ?? '') + name); - } - found.name = name; - found.folder = folder; - found.container = container; - found.size = fileData.byteLength; - return (await File.getInstance().findByIdAndUpdate(found._id, found)) as File; } } diff --git a/modules/storage/src/interfaces/IStorageProvider.ts b/modules/storage/src/interfaces/IStorageProvider.ts index ca3d50b82..ca956b241 100644 --- a/modules/storage/src/interfaces/IStorageProvider.ts +++ b/modules/storage/src/interfaces/IStorageProvider.ts @@ -37,23 +37,6 @@ export interface IStorageProvider { getSignedUrl(fileName: string): Promise; getPublicUrl(fileName: string): Promise; - getUploadUrl(fileName: string): Promise; - - rename(currentFilename: string, newFilename: string): Promise; - - moveToFolder(filename: string, newFolder: string): Promise; - moveToFolderAndRename( - currentFilename: string, - newFilename: string, - newFolder: string, - ): Promise; - - moveToContainer(filename: string, newContainer: string): Promise; - - moveToContainerAndRename( - currentFilename: string, - newFilename: string, - newContainer: string, - ): Promise; + getUploadUrl(fileName: string): Promise; } diff --git a/modules/storage/src/providers/aliyun/index.ts b/modules/storage/src/providers/aliyun/index.ts index 13727a5f2..328969278 100644 --- a/modules/storage/src/providers/aliyun/index.ts +++ b/modules/storage/src/providers/aliyun/index.ts @@ -1,7 +1,6 @@ import { IStorageProvider, StorageConfig } from '../../interfaces'; import OSS from 'ali-oss'; import fs from 'fs'; -import { basename } from 'path'; import ConduitGrpcSdk from '@conduitplatform/grpc-sdk'; import { SIGNED_URL_EXPIRY_SECONDS } from '../../constants/expiry'; @@ -153,61 +152,4 @@ export class AliyunStorage implements IStorageProvider { return url; } - - async rename(currentFilename: string, newFilename: string): Promise { - await this._ossClient.copy(newFilename, currentFilename); - await this.delete(currentFilename); - return true; - } - - async moveToFolder(filename: string, newFolder: string): Promise { - await this._ossClient.copy(newFolder + basename(filename), filename); - await this.delete(filename); - return true; - } - - async moveToFolderAndRename( - currentFilename: string, - newFilename: string, - newFolder: string, - ): Promise { - await this._ossClient.copy(newFolder + newFilename, currentFilename); - await this.delete(currentFilename); - return true; - } - - async moveToContainer( - filename: string, - newContainer: string, - ): Promise { - const oldBucket = this._activeContainer; - - this._ossClient.useBucket(newContainer); - - await this._ossClient.copy(filename, filename, oldBucket); - - this._ossClient.useBucket(oldBucket); - - await this.delete(filename); - - return true; - } - - async moveToContainerAndRename( - currentFilename: string, - newFilename: string, - newContainer: string, - ): Promise { - const oldBucket = this._activeContainer; - - this._ossClient.useBucket(newContainer); - - await this._ossClient.copy(newFilename, currentFilename, oldBucket); - - this._ossClient.useBucket(oldBucket); - - await this.delete(currentFilename); - - return true; - } } diff --git a/modules/storage/src/providers/aws/index.ts b/modules/storage/src/providers/aws/index.ts index f946d7776..deea28318 100644 --- a/modules/storage/src/providers/aws/index.ts +++ b/modules/storage/src/providers/aws/index.ts @@ -219,37 +219,6 @@ export class AWSS3Storage implements IStorageProvider { return `https://${this._activeContainer}.s3.amazonaws.com/${fileName}`; } - async rename(currentFilename: string, newFilename: string): Promise { - throw new Error('Not implemented'); - } - - async moveToFolder(filename: string, newFolder: string): Promise { - throw new Error('Method not implemented.'); - } - - async moveToFolderAndRename( - currentFilename: string, - newFilename: string, - newFolder: string, - ): Promise { - throw new Error('Method not implemented.'); - } - - async moveToContainer( - filename: string, - newContainer: string, - ): Promise { - throw new Error('Method not implemented.'); - } - - async moveToContainerAndRename( - currentFilename: string, - newFilename: string, - newContainer: string, - ): Promise { - throw new Error('Method not implemented.'); - } - private async listFiles(name: string) { const files = await this._storage.send( new ListObjectsCommand({ diff --git a/modules/storage/src/providers/azure/index.ts b/modules/storage/src/providers/azure/index.ts index c39f1735a..917a7c8bf 100644 --- a/modules/storage/src/providers/azure/index.ts +++ b/modules/storage/src/providers/azure/index.ts @@ -9,7 +9,7 @@ import { import fs from 'fs'; import ConduitGrpcSdk from '@conduitplatform/grpc-sdk'; import { streamToBuffer } from '../../utils'; -import { SIGNED_URL_EXPIRY, SIGNED_URL_EXPIRY_DATE } from '../../constants/expiry'; +import { SIGNED_URL_EXPIRY_DATE } from '../../constants/expiry'; export class AzureStorage implements IStorageProvider { _activeContainer: string = ''; @@ -165,30 +165,6 @@ export class AzureStorage implements IStorageProvider { return true; } - async rename(currentFilename: string, newFilename: string): Promise { - // await this._storage.getContainerClient(this._activeContainer).getBlockBlobClient(currentFilename).move(newFilename); - // return true; - throw new Error('Not Implemented yet!'); - } - - async moveToFolder(filename: string, newFolder: string): Promise { - // let newBucketFile = this._storage.getContainerClient(newFolder).file(filename) - // await this._storage.getContainerClient(this._activeContainer).file(filename).move(newBucketFile); - // return true; - throw new Error('Not Implemented yet!'); - } - - async moveToFolderAndRename( - currentFilename: string, - newFilename: string, - newFolder: string, - ): Promise { - // let newBucketFile = this._storage.getContainerClient(newFolder).file(newFilename) - // await this._storage.getContainerClient(this._activeContainer).file(currentFilename).move(newBucketFile); - // return true; - throw new Error('Not Implemented yet!'); - } - async containerExists(name: string): Promise { return await this._storage.getContainerClient(name).exists(); } @@ -200,21 +176,6 @@ export class AzureStorage implements IStorageProvider { return true; } - async moveToContainer( - filename: string, - newContainer: string, - ): Promise { - throw new Error('Not Implemented yet!'); - } - - async moveToContainerAndRename( - currentFilename: string, - newFilename: string, - newContainer: string, - ): Promise { - throw new Error('Not Implemented yet!'); - } - getUploadUrl(fileName: string): Promise { const containerClient = this._storage.getContainerClient(this._activeContainer); const sasOptions: BlobSASSignatureValues = { diff --git a/modules/storage/src/providers/google/index.ts b/modules/storage/src/providers/google/index.ts index d7e216c75..67532b0b2 100644 --- a/modules/storage/src/providers/google/index.ts +++ b/modules/storage/src/providers/google/index.ts @@ -43,28 +43,6 @@ export class GoogleCloudStorage implements IStorageProvider { return exists[0]; } - async moveToContainer( - filename: string, - newContainer: string, - ): Promise { - const newBucketFile = this._storage.bucket(newContainer).file(filename); - await this._storage.bucket(this._activeBucket).file(filename).move(newBucketFile); - return true; - } - - async moveToContainerAndRename( - currentFilename: string, - newFilename: string, - newContainer: string, - ): Promise { - const newBucketFile = this._storage.bucket(newContainer).file(newFilename); - await this._storage - .bucket(this._activeBucket) - .file(currentFilename) - .move(newBucketFile); - return true; - } - /** * Used to create a new folder * @param name For the folder @@ -158,26 +136,6 @@ export class GoogleCloudStorage implements IStorageProvider { return true; } - async rename(currentFilename: string, newFilename: string): Promise { - await this._storage - .bucket(this._activeBucket) - .file(currentFilename) - .move(newFilename); - return true; - } - - async moveToFolder(filename: string, newFolder: string): Promise { - throw new Error('Method not implemented!'); - } - - async moveToFolderAndRename( - currentFilename: string, - newFilename: string, - newFolder: string, - ): Promise { - throw new Error('Method not implemented!'); - } - getUploadUrl(fileName: string): Promise { return this._storage .bucket(this._activeBucket) diff --git a/modules/storage/src/providers/local/index.ts b/modules/storage/src/providers/local/index.ts index 5c8d7cd66..af52bf5e8 100644 --- a/modules/storage/src/providers/local/index.ts +++ b/modules/storage/src/providers/local/index.ts @@ -7,11 +7,9 @@ import { existsSync, mkdir, readFile, - rename, unlink, writeFile, rmSync, - statSync, } from 'fs'; import { resolve } from 'path'; import ConduitGrpcSdk from '@conduitplatform/grpc-sdk'; @@ -174,50 +172,6 @@ export class LocalStorage implements IStorageProvider { }); } - rename(currentFilename: string, newFilename: string): Promise { - const self = this; - const path = self._storagePath + '/' + self._activeContainer + '/'; - - return new Promise(function (res, reject) { - rename(resolve(path, currentFilename), resolve(path, newFilename), function (err) { - if (err) reject(err); - else res(true); - }); - }); - } - - moveToFolder(filename: string, newFolder: string): Promise { - const self = this; - return new Promise(function (res, reject) { - rename( - resolve(self._storagePath, filename), - resolve(self._rootStoragePath, newFolder, filename), - function (err) { - if (err) reject(err); - else res(true); - }, - ); - }); - } - - moveToFolderAndRename( - currentFilename: string, - newFilename: string, - newFolder: string, - ): Promise { - const self = this; - return new Promise(function (res, reject) { - rename( - resolve(self._storagePath, currentFilename), - resolve(self._rootStoragePath, newFolder, newFilename), - function (err) { - if (err) reject(err); - else res(true); - }, - ); - }); - } - getPublicUrl(fileName: string): Promise { throw new Error('Method not implemented!'); } @@ -239,16 +193,4 @@ export class LocalStorage implements IStorageProvider { this._activeContainer = name; return this.createFolder(name); } - - moveToContainer(filename: string, newContainer: string): Promise { - throw new Error('Method not implemented!| Error'); - } - - moveToContainerAndRename( - currentFilename: string, - newFilename: string, - newContainer: string, - ): Promise { - throw new Error('Method not implemented!| Error'); - } } diff --git a/modules/storage/src/routes/index.ts b/modules/storage/src/routes/index.ts index 25dd15a0e..f77868053 100644 --- a/modules/storage/src/routes/index.ts +++ b/modules/storage/src/routes/index.ts @@ -1,7 +1,9 @@ import { FileHandlers } from '../handlers/file'; import ConduitGrpcSdk, { + ConduitNumber, ConduitRouteActions, ConduitRouteReturnDefinition, + ConduitString, GrpcServer, RoutingManager, TYPE, @@ -82,14 +84,33 @@ export class StorageRoutes { isPublic: TYPE.Boolean, }, action: ConduitRouteActions.POST, - path: '/storage/fileByUrl', + path: '/storage/upload', description: `Creates a new file and provides a URL to upload it to.`, middlewares: ['authMiddleware'], }, new ConduitRouteReturnDefinition('CreateFileByUrl', File.name), this.fileHandlers.createFileUploadUrl.bind(this.fileHandlers), ); - + this._routingManager.route( + { + urlParams: { + id: { type: TYPE.String, required: true }, + }, + bodyParams: { + name: ConduitString.Optional, + folder: ConduitString.Optional, + container: ConduitString.Optional, + mimeType: ConduitString.Optional, + size: ConduitNumber.Optional, + }, + action: ConduitRouteActions.PATCH, + path: '/storage/upload/:id', + description: `Updates a file and provides a URL to upload its data to.`, + middlewares: ['authMiddleware'], + }, + new ConduitRouteReturnDefinition('PatchFileByUrl', 'String'), + this.fileHandlers.updateFileUploadUrl.bind(this.fileHandlers), + ); this._routingManager.route( { urlParams: { @@ -128,18 +149,18 @@ export class StorageRoutes { id: { type: TYPE.String, required: true }, }, bodyParams: { - name: TYPE.String, - mimeType: TYPE.String, - data: TYPE.String, - folder: TYPE.String, - container: TYPE.String, + name: ConduitString.Optional, + folder: ConduitString.Optional, + container: ConduitString.Optional, + data: ConduitString.Required, + mimeType: ConduitString.Optional, }, action: ConduitRouteActions.PATCH, path: '/storage/file/:id', description: `Updates a file.`, middlewares: ['authMiddleware'], }, - new ConduitRouteReturnDefinition('FileUpdateResponse', File.name), + new ConduitRouteReturnDefinition('PatchFile', File.name), this.fileHandlers.updateFile.bind(this.fileHandlers), ); }