feat(storage): add file uploads by url instead of base64

fix(storage): add consistent expiry of signed urls

modules/storage/src/admin/index.ts

@@ -10,6 +10,7 @@ import ConduitGrpcSdk, {
   Query,
   RouteOptionType,
   RoutingManager,
+  TYPE,
   UnparsedRouterResponse,
 } from '@conduitplatform/grpc-sdk';
 import { status } from '@grpc/grpc-js';
@@ -80,6 +81,23 @@ export class AdminRoutes {
       new ConduitRouteReturnDefinition('CreateFile', File.name),
       this.fileHandlers.createFile.bind(this.fileHandlers),
     );
+    this.routingManager.route(
+      {
+        bodyParams: {
+          name: { type: TYPE.String, required: true },
+          mimeType: TYPE.String,
+          folder: { type: TYPE.String, required: false },
+          size: { type: TYPE.Number, required: false },
+          container: { type: TYPE.String, required: false },
+          isPublic: TYPE.Boolean,
+        },
+        action: ConduitRouteActions.POST,
+        path: '/files/uploadByUrl',
+        description: `Creates a new file and provides a URL to upload it to.`,
+      },
+      new ConduitRouteReturnDefinition('CreateFileByUrl', File.name),
+      this.fileHandlers.createFileUploadUrl.bind(this.fileHandlers),
+    );
     this.routingManager.route(
       {
         path: '/files/:id',

modules/storage/src/constants/expiry.ts

@@ -0,0 +1,4 @@
+// 60 minutes
+export const SIGNED_URL_EXPIRY = 3600 * 1000;
+export const SIGNED_URL_EXPIRY_DATE = () => Date.now() + SIGNED_URL_EXPIRY;
+export const SIGNED_URL_EXPIRY_SECONDS = SIGNED_URL_EXPIRY / 1000;

modules/storage/src/handlers/file.ts

@@ -79,6 +79,48 @@ export class FileHandlers {
     }
   }
 
+  async createFileUploadUrl(call: ParsedRouterRequest): Promise<UnparsedRouterResponse> {
+    const { name, folder, container, size = 0, mimeType, isPublic } = call.request.params;
+    let newFolder;
+    if (isNil(folder)) {
+      newFolder = '/';
+    } else {
+      newFolder = folder.trim().slice(-1) !== '/' ? folder.trim() + '/' : folder.trim();
+    }
+    const config = ConfigController.getInstance().config;
+    const usedContainer = isNil(container)
+      ? config.defaultContainer
+      : await this.findOrCreateContainer(container, isPublic);
+    if (!isNil(folder)) {
+      await this.findOrCreateFolder(newFolder, usedContainer, isPublic);
+    }
+
+    const exists = await File.getInstance().findOne({
+      name,
+      container: usedContainer,
+      folder: newFolder,
+    });
+    if (exists) {
+      throw new GrpcError(status.ALREADY_EXISTS, 'File already exists');
+    }
+
+    try {
+      return this.getFileUploadUrl(
+        usedContainer,
+        newFolder,
+        isPublic,
+        name,
+        size,
+        mimeType,
+      );
+    } catch (e) {
+      throw new GrpcError(
+        status.INTERNAL,
+        (e as Error).message ?? 'Something went wrong',
+      );
+    }
+  }
+
   async updateFile(call: ParsedRouterRequest): Promise<UnparsedRouterResponse> {
     const { id, data, name, container, folder, mimeType } = call.request.params;
     try {
@@ -294,6 +336,38 @@ export class FileHandlers {
     });
   }
 
+  private async getFileUploadUrl(
+    container: string,
+    folder: string,
+    isPublic: boolean,
+    name: string,
+    size: number,
+    mimeType: string,
+  ): Promise<string> {
+    await this.storageProvider
+      .container(container)
+      .store((folder ?? '') + name, Buffer.from('PENDING UPLOAD'), isPublic);
+    const publicUrl = isPublic
+      ? await this.storageProvider
+          .container(container)
+          .getPublicUrl((folder ?? '') + name)
+      : null;
+    ConduitGrpcSdk.Metrics?.increment('files_total');
+    ConduitGrpcSdk.Metrics?.increment('storage_size_bytes_total', size);
+    await File.getInstance().create({
+      name,
+      mimeType,
+      folder: folder,
+      container: container,
+      isPublic,
+      url: publicUrl,
+    });
+
+    return (await this.storageProvider
+      .container(container)
+      .getUploadUrl((folder ?? '') + name)) as string;
+  }
+
   private async storeUpdatedFile(
     container: string,
     folder: string,

modules/storage/src/interfaces/IStorageProvider.ts

@@ -37,6 +37,7 @@ export interface IStorageProvider {
   getSignedUrl(fileName: string): Promise<any | Error>;
 
   getPublicUrl(fileName: string): Promise<any | Error>;
+  getUploadUrl(fileName: string): Promise<string | Error>;
 
   rename(currentFilename: string, newFilename: string): Promise<boolean | Error>;
 

modules/storage/src/providers/aliyun/index.ts

@@ -3,6 +3,7 @@ import OSS from 'ali-oss';
 import fs from 'fs';
 import { basename } from 'path';
 import ConduitGrpcSdk from '@conduitplatform/grpc-sdk';
+import { SIGNED_URL_EXPIRY_SECONDS } from '../../constants/expiry';
 
 export class AliyunStorage implements IStorageProvider {
   private _activeContainer: string = '';
@@ -21,6 +22,15 @@ export class AliyunStorage implements IStorageProvider {
     });
   }
 
+  getUploadUrl(fileName: string): Promise<string | Error> {
+    const url = this._ossClient.signatureUrl(fileName, {
+      expires: SIGNED_URL_EXPIRY_SECONDS,
+      method: 'PUT',
+    });
+
+    return Promise.resolve(url);
+  }
+
   async containerExists(name: string): Promise<boolean | Error> {
     return await this._ossClient
       .getBucketInfo(name)
@@ -128,7 +138,7 @@ export class AliyunStorage implements IStorageProvider {
 
   async getSignedUrl(fileName: string): Promise<any | Error> {
     const url = this._ossClient.signatureUrl(fileName, {
-      expires: 3600,
+      expires: SIGNED_URL_EXPIRY_SECONDS,
       method: 'GET',
     });
 

modules/storage/src/providers/aws/index.ts

@@ -16,6 +16,7 @@ import { streamToBuffer } from '../../utils';
 import fs from 'fs';
 import { getSignedUrl as awsGetSignedUrl } from '@aws-sdk/s3-request-presigner';
 import ConduitGrpcSdk, { ConfigController } from '@conduitplatform/grpc-sdk';
+import { SIGNED_URL_EXPIRY_SECONDS } from '../../constants/expiry';
 
 type AwsError = { $metadata: { httpStatusCode: number } };
 type GetResult = Buffer | Error;
@@ -209,7 +210,9 @@ export class AWSS3Storage implements IStorageProvider {
       Bucket: this._activeContainer,
       Key: fileName,
     });
-    return await awsGetSignedUrl(this._storage, command);
+    return awsGetSignedUrl(this._storage, command, {
+      expiresIn: SIGNED_URL_EXPIRY_SECONDS,
+    });
   }
 
   async getPublicUrl(fileName: string) {
@@ -257,4 +260,14 @@ export class AWSS3Storage implements IStorageProvider {
     if (!files.Contents) return [];
     return files.Contents;
   }
+
+  getUploadUrl(fileName: string): Promise<string | Error> {
+    const command = new PutObjectCommand({
+      Bucket: this._activeContainer,
+      Key: fileName,
+    });
+    return awsGetSignedUrl(this._storage, command, {
+      expiresIn: SIGNED_URL_EXPIRY_SECONDS,
+    });
+  }
 }

modules/storage/src/providers/azure/index.ts

@@ -9,6 +9,7 @@ import {
 import fs from 'fs';
 import ConduitGrpcSdk from '@conduitplatform/grpc-sdk';
 import { streamToBuffer } from '../../utils';
+import { SIGNED_URL_EXPIRY, SIGNED_URL_EXPIRY_DATE } from '../../constants/expiry';
 
 export class AzureStorage implements IStorageProvider {
   _activeContainer: string = '';
@@ -126,7 +127,7 @@ export class AzureStorage implements IStorageProvider {
     const sasOptions: BlobSASSignatureValues = {
       containerName: containerClient.containerName,
       blobName: fileName,
-      expiresOn: new Date(new Date().valueOf() + 3600 * 1000),
+      expiresOn: new Date(SIGNED_URL_EXPIRY_DATE()),
       permissions: BlobSASPermissions.parse('r'),
     };
     return this.blobClient(fileName).generateSasUrl(sasOptions);
@@ -213,4 +214,15 @@ export class AzureStorage implements IStorageProvider {
   ): Promise<boolean | Error> {
     throw new Error('Not Implemented yet!');
   }
+
+  getUploadUrl(fileName: string): Promise<string> {
+    const containerClient = this._storage.getContainerClient(this._activeContainer);
+    const sasOptions: BlobSASSignatureValues = {
+      containerName: containerClient.containerName,
+      blobName: fileName,
+      expiresOn: new Date(SIGNED_URL_EXPIRY_DATE()),
+      permissions: BlobSASPermissions.from({ read: true, create: true, write: true }),
+    };
+    return this.blobClient(fileName).generateSasUrl(sasOptions);
+  }
 }

modules/storage/src/providers/google/index.ts

@@ -1,6 +1,7 @@
 import { IStorageProvider, StorageConfig } from '../../interfaces';
 import { Storage } from '@google-cloud/storage';
 import ConduitGrpcSdk from '@conduitplatform/grpc-sdk';
+import { SIGNED_URL_EXPIRY_DATE } from '../../constants/expiry';
 
 /**
  * WARNING: DO NOT USE THIS, IT NEEDS A REWRITE
@@ -19,6 +20,7 @@ export class GoogleCloudStorage implements IStorageProvider {
   deleteContainer(name: string): Promise<boolean | Error> {
     throw new Error('Method not implemented.');
   }
+
   deleteFolder(name: string): Promise<boolean | Error> {
     throw new Error('Method not implemented.');
   }
@@ -129,7 +131,7 @@ export class GoogleCloudStorage implements IStorageProvider {
       .file(fileName)
       .getSignedUrl({
         action: 'read',
-        expires: Date.now() + 14400000,
+        expires: SIGNED_URL_EXPIRY_DATE(),
       })
       .then((r: any) => {
         if (r.data && r.data[0]) {
@@ -175,4 +177,20 @@ export class GoogleCloudStorage implements IStorageProvider {
   ): Promise<boolean | Error> {
     throw new Error('Method not implemented!');
   }
+
+  getUploadUrl(fileName: string): Promise<string | Error> {
+    return this._storage
+      .bucket(this._activeBucket)
+      .file(fileName)
+      .getSignedUrl({
+        action: 'write',
+        expires: SIGNED_URL_EXPIRY_DATE(),
+      })
+      .then((r: any) => {
+        if (r.data && r.data[0]) {
+          return r.data[0];
+        }
+        return r;
+      });
+  }
 }

modules/storage/src/providers/local/index.ts

@@ -39,6 +39,10 @@ export class LocalStorage implements IStorageProvider {
     }
   }
 
+  getUploadUrl(fileName: string): Promise<string | Error> {
+    throw new Error('Method not implemented.');
+  }
+
   deleteContainer(name: string): Promise<boolean | Error> {
     return this.deleteFolder(name);
   }

modules/storage/src/routes/index.ts

@@ -71,6 +71,24 @@ export class StorageRoutes {
         new ConduitRouteReturnDefinition('CreateFile', File.name),
         this.fileHandlers.createFile.bind(this.fileHandlers),
       );
+      this._routingManager.route(
+        {
+          bodyParams: {
+            name: { type: TYPE.String, required: true },
+            mimeType: TYPE.String,
+            folder: { type: TYPE.String, required: false },
+            size: { type: TYPE.Number, required: false },
+            container: { type: TYPE.String, required: false },
+            isPublic: TYPE.Boolean,
+          },
+          action: ConduitRouteActions.POST,
+          path: '/storage/fileByUrl',
+          description: `Creates a new file and provides a URL to upload it to.`,
+          middlewares: ['authMiddleware'],
+        },
+        new ConduitRouteReturnDefinition('CreateFileByUrl', File.name),
+        this.fileHandlers.createFileUploadUrl.bind(this.fileHandlers),
+      );
 
       this._routingManager.route(
         {