Skip to content

Commit

Permalink
feat(storage)!: update file by url endpoints & remove of rename/move …
Browse files Browse the repository at this point in the history 
…support (#552)
  • Loading branch information
@ChrisPdgn @kkopanidis
ChrisPdgn authored and kkopanidis committed Mar 21, 2023
1 parent bfd059a commit 68c473a
Show file tree
Hide file tree
Showing 9 changed files with 211 additions and 337 deletions.
23 changes: 21 additions & 2 deletions modules/storage/src/admin/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ export class AdminRoutes {
isPublic: TYPE.Boolean,
},
action: ConduitRouteActions.POST,
path: '/files/uploadByUrl',
path: '/files/upload',
description: `Creates a new file and provides a URL to upload it to.`,
},
new ConduitRouteReturnDefinition('CreateFileByUrl', File.name),
Expand All @@ -108,15 +108,34 @@ export class AdminRoutes {
},
bodyParams: {
name: ConduitString.Optional,
data: ConduitString.Optional,
folder: ConduitString.Optional,
container: ConduitString.Optional,
data: ConduitString.Required,
mimeType: ConduitString.Optional,
},
},
new ConduitRouteReturnDefinition('PatchFile', File.name),
this.fileHandlers.updateFile.bind(this.fileHandlers),
);
this.routingManager.route(
{
urlParams: {
id: { type: TYPE.String, required: true },
},
bodyParams: {
name: ConduitString.Optional,
folder: ConduitString.Optional,
container: ConduitString.Optional,
mimeType: ConduitString.Optional,
size: ConduitNumber.Optional,
},
action: ConduitRouteActions.PATCH,
path: '/files/upload/:id',
description: `Updates a file and provides a URL to upload its data to.`,
},
new ConduitRouteReturnDefinition('PatchFileByUrl', 'String'),
this.fileHandlers.updateFileUploadUrl.bind(this.fileHandlers),
);
this.routingManager.route(
{
path: '/files/:id',
Expand Down
239 changes: 159 additions & 80 deletions modules/storage/src/handlers/file.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ export class FileHandlers {
}

try {
return this.getFileUploadUrl(
return await this._createFileUploadUrl(
usedContainer,
newFolder,
isPublic,
Expand All @@ -121,56 +121,51 @@ export class FileHandlers {
}
}

async updateFile(call: ParsedRouterRequest): Promise<UnparsedRouterResponse> {
const { id, data, name, container, folder, mimeType } = call.request.params;
async updateFileUploadUrl(call: ParsedRouterRequest): Promise<UnparsedRouterResponse> {
const { id, mimeType, size } = call.request.params;
const found = await File.getInstance().findOne({ _id: id });
if (isNil(found)) {
throw new GrpcError(status.NOT_FOUND, 'File does not exist');
}
const { name, folder, container } = await this.validateFilenameAndContainer(
call,
found,
);
try {
const found = await File.getInstance().findOne({ _id: id });
if (isNil(found)) {
throw new GrpcError(status.NOT_FOUND, 'File does not exist');
}
let fileData = await this.storageProvider
.container(found.container)
.get((found.folder ?? '') + found.name);

if (!isNil(data)) {
fileData = Buffer.from(data, 'base64');
}

const newName = name ?? found.name;
let newFolder = folder ?? found.folder;
if (!newFolder.endsWith('/')) {
// existing folder names are currently suffixed by "/" upon creation
newFolder += '/';
}
const newContainer = container ?? found.container;
found.mimeType = mimeType ?? found.mimeType;
const isDataUpdate =
newName === found.name &&
newContainer === found.container &&
newFolder === found.folder;

if (newContainer !== found.container) {
await this.findOrCreateContainer(newContainer);
}
if (newFolder !== found.folder) {
await this.findOrCreateFolder(newFolder, newContainer);
}
return await this._updateFileUploadUrl(
name,
folder,
container,
mimeType ?? found.mimeType,
found,
size,
);
} catch (e) {
throw new GrpcError(
status.INTERNAL,
(e as Error).message ?? 'Something went wrong',
);
}
}

const exists = await File.getInstance().findOne({
name: newName,
container: newContainer,
folder: newFolder,
});
if (!isDataUpdate && exists) {
throw new GrpcError(status.ALREADY_EXISTS, 'File already exists');
}
return this.storeUpdatedFile(
newContainer,
newFolder,
newName,
async updateFile(call: ParsedRouterRequest): Promise<UnparsedRouterResponse> {
const { id, data, mimeType } = call.request.params;
const found = await File.getInstance().findOne({ _id: id });
if (isNil(found)) {
throw new GrpcError(status.NOT_FOUND, 'File does not exist');
}
const { name, folder, container } = await this.validateFilenameAndContainer(
call,
found,
);
try {
return await this._updateFile(
name,
folder,
container,
Buffer.from(data, 'base64'),
mimeType ?? found.mimeType,
found,
fileData,
isDataUpdate,
);
} catch (e) {
throw new GrpcError(
Expand All @@ -191,7 +186,7 @@ export class FileHandlers {
}
const success = await this.storageProvider
.container(found.container)
.delete((found.folder ?? '') + found.name);
.delete((found.folder === '/' ? '' : found.folder) + found.name);
if (!success) {
throw new GrpcError(status.INTERNAL, 'File could not be deleted');
}
Expand All @@ -218,7 +213,7 @@ export class FileHandlers {
}
const url = await this.storageProvider
.container(found.container)
.getSignedUrl((found.folder ?? '') + found.name);
.getSignedUrl((found.folder === '/' ? '' : found.folder) + found.name);

if (!call.request.params.redirect) {
return { result: url };
Expand All @@ -245,7 +240,9 @@ export class FileHandlers {
let data: Buffer;
const result = await this.storageProvider
.container(file.container)
.get(file.folder ? file.folder + file.name : file.name);
.get(
file.folder ? (file.folder === '/' ? '' : file.folder) + file.name : file.name,
);
if (result instanceof Error) {
throw result;
} else {
Expand Down Expand Up @@ -314,14 +311,13 @@ export class FileHandlers {
): Promise<File> {
const buffer = Buffer.from(data, 'base64');
const size = buffer.byteLength;

await this.storageProvider
.container(container)
.store((folder ?? '') + name, buffer, isPublic);
.store((folder === '/' ? '' : folder) + name, buffer, isPublic);
const publicUrl = isPublic
? await this.storageProvider
.container(container)
.getPublicUrl((folder ?? '') + name)
.getPublicUrl((folder === '/' ? '' : folder) + name)
: null;
ConduitGrpcSdk.Metrics?.increment('files_total');
ConduitGrpcSdk.Metrics?.increment('storage_size_bytes_total', size);
Expand All @@ -336,7 +332,7 @@ export class FileHandlers {
});
}

private async getFileUploadUrl(
private async _createFileUploadUrl(
container: string,
folder: string,
isPublic: boolean,
Expand All @@ -346,11 +342,15 @@ export class FileHandlers {
): Promise<string> {
await this.storageProvider
.container(container)
.store((folder ?? '') + name, Buffer.from('PENDING UPLOAD'), isPublic);
.store(
(folder === '/' ? '' : folder) + name,
Buffer.from('PENDING UPLOAD'),
isPublic,
);
const publicUrl = isPublic
? await this.storageProvider
.container(container)
.getPublicUrl((folder ?? '') + name)
.getPublicUrl((folder === '/' ? '' : folder) + name)
: null;
ConduitGrpcSdk.Metrics?.increment('files_total');
ConduitGrpcSdk.Metrics?.increment('storage_size_bytes_total', size);
Expand All @@ -362,43 +362,122 @@ export class FileHandlers {
isPublic,
url: publicUrl,
});

return (await this.storageProvider
.container(container)
.getUploadUrl((folder ?? '') + name)) as string;
.getUploadUrl((folder === '/' ? '' : folder) + name)) as string;
}

private async storeUpdatedFile(
container: string,
private async validateFilenameAndContainer(call: ParsedRouterRequest, file: File) {
const { name, folder, container } = call.request.params;
const newName = name ?? file.name;
const newContainer = container ?? file.container;
if (newContainer !== file.container) {
await this.findOrCreateContainer(newContainer);
}
// Existing folder names are currently suffixed by "/" upon creation
const newFolder = isNil(folder)
? file.folder
: folder.trim().slice(-1) !== '/'
? folder.trim() + '/'
: folder.trim();
if (newFolder !== file.folder) {
await this.findOrCreateFolder(newFolder, newContainer);
}
const exists = await File.getInstance().findOne({
name: newName,
container: newContainer,
folder: newFolder,
});
if (!isNil(exists)) {
throw new GrpcError(status.ALREADY_EXISTS, 'File already exists');
}
return {
name: newName,
folder: newFolder,
container: newContainer,
};
}

private async _updateFileUploadUrl(
name: string,
folder: string,
container: string,
mimeType: string,
file: File,
size: number | undefined | null,
): Promise<string> {
const onlyDataUpdate =
name === file.name && folder === file.folder && container === file.container;
if (onlyDataUpdate) {
await File.getInstance().findByIdAndUpdate(file._id, { mimeType });
} else {
await this.storageProvider
.container(container)
.store(
(folder === '/' ? '' : folder) + name,
Buffer.from('PENDING UPLOAD'),
file.isPublic,
);
await this.storageProvider
.container(file.container)
.delete((file.folder === '/' ? '' : file.folder) + file.name);
const url = file.isPublic
? await this.storageProvider
.container(container)
.getPublicUrl((folder === '/' ? '' : folder) + name)
: null;
await File.getInstance().findByIdAndUpdate(file._id, {
name,
folder,
container,
url,
mimeType,
});
}
if (!isNil(size)) this.updateFileMetrics(file.size, size!);
return (await this.storageProvider
.container(container)
.getUploadUrl((folder === '/' ? '' : folder) + name)) as string;
}

private async _updateFile(
name: string,
found: File,
fileData: any,
isDataUpdate: boolean,
folder: string,
container: string,
data: Buffer,
mimeType: string,
file: File,
): Promise<File> {
const onlyDataUpdate =
name === file.name && folder === file.folder && container === file.container;
await this.storageProvider
.container(container)
.store((folder ?? '') + name, fileData);
// calling delete after store call succeeds
if (!isDataUpdate) {
.store((folder === '/' ? '' : folder) + name, data, file.isPublic);
if (!onlyDataUpdate) {
await this.storageProvider
.container(found.container)
.delete((found.folder ?? '') + found.name);
.container(file.container)
.delete((file.folder === '/' ? '' : file.folder) + file.name);
}
const url = file.isPublic
? await this.storageProvider
.container(container)
.getPublicUrl((folder === '/' ? '' : folder) + name)
: null;
const updatedFile = (await File.getInstance().findByIdAndUpdate(file._id, {
name,
folder,
container,
url,
mimeType,
})) as File;
this.updateFileMetrics(file.size, data.byteLength);
return updatedFile;
}

const fileSizeDiff = Math.abs(found.size - fileData.byteLength);
private updateFileMetrics(currentSize: number, newSize: number) {
const fileSizeDiff = Math.abs(currentSize - newSize);
fileSizeDiff < 0
? ConduitGrpcSdk.Metrics?.increment('storage_size_bytes_total', fileSizeDiff)
: ConduitGrpcSdk.Metrics?.decrement('storage_size_bytes_total', fileSizeDiff);
if (found.isPublic) {
found.url = await this.storageProvider
.container(container)
.getPublicUrl((folder ?? '') + name);
}
found.name = name;
found.folder = folder;
found.container = container;
found.size = fileData.byteLength;
return (await File.getInstance().findByIdAndUpdate(found._id, found)) as File;
}
}
19 changes: 1 addition & 18 deletions modules/storage/src/interfaces/IStorageProvider.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,23 +37,6 @@ export interface IStorageProvider {
getSignedUrl(fileName: string): Promise<any | Error>;

getPublicUrl(fileName: string): Promise<any | Error>;
getUploadUrl(fileName: string): Promise<string | Error>;

rename(currentFilename: string, newFilename: string): Promise<boolean | Error>;

moveToFolder(filename: string, newFolder: string): Promise<boolean | Error>;

moveToFolderAndRename(
currentFilename: string,
newFilename: string,
newFolder: string,
): Promise<boolean | Error>;

moveToContainer(filename: string, newContainer: string): Promise<boolean | Error>;

moveToContainerAndRename(
currentFilename: string,
newFilename: string,
newContainer: string,
): Promise<boolean | Error>;
getUploadUrl(fileName: string): Promise<string | Error>;
}
Loading

0 comments on commit 68c473a

Please sign in to comment.