fix: implement cloud file operations

examples/express-gcs.ts

@@ -5,10 +5,10 @@ const app = express();
 
 const storage = new GCStorage({ maxUploadSize: '1GB' });
 
-storage.onComplete = ({ uri, id }) => {
-  console.log(`File upload complete, storage path: ${uri}`);
-  // send gcs link to client
-  return { id, link: uri };
+storage.onComplete = async file => {
+  const info = await file.get().catch(console.error);
+  console.log(info);
+  return file.move(file.originalName).catch(console.error);
 };
 
 app.use('/files', uploadx({ storage }));

packages/gcs/src/gcs-storage.ts

@@ -21,6 +21,7 @@ import * as http from 'http';
 import request from 'node-fetch';
 import { authScopes, BUCKET_NAME, storageAPI, uploadAPI } from './constants';
 import { GCSMetaStorage, GCSMetaStorageOptions } from './gcs-meta-storage';
+import { resolve } from 'url';
 
 export interface ClientError extends Error {
   code: string;
@@ -72,9 +73,13 @@ export interface GCStorageOptions extends BaseStorageOptions<GCSFile>, GoogleAut
   metaStorageConfig?: LocalMetaStorageOptions | GCSMetaStorageOptions;
 }
 
-export class GCSFile extends File {
+export interface GCSFile extends File {
   GCSUploadURI?: string;
-  uri = '';
+  uri: string;
+  move: (dest: string) => Promise<Record<string, string>>;
+  copy: (dest: string) => Promise<Record<string, string>>;
+  get: () => Promise<Record<string, string>>;
+  delete: () => Promise<any>;
 }
 
 /**
@@ -95,6 +100,7 @@ export class GCStorage extends BaseStorage<GCSFile> {
   storageBaseURI: string;
   uploadBaseURI: string;
   meta: MetaStorage<GCSFile>;
+  private readonly bucket: string;
 
   constructor(public config: GCStorageOptions = {}) {
     super(config);
@@ -109,11 +115,11 @@ export class GCStorage extends BaseStorage<GCSFile> {
     }
     config.scopes ||= authScopes;
     config.keyFile ||= process.env.GCS_KEYFILE;
-    const bucketName = config.bucket || process.env.GCS_BUCKET || BUCKET_NAME;
-    this.storageBaseURI = [storageAPI, bucketName, 'o'].join('/');
-    this.uploadBaseURI = [uploadAPI, bucketName, 'o'].join('/');
+    this.bucket = config.bucket || process.env.GCS_BUCKET || BUCKET_NAME;
+    this.storageBaseURI = [storageAPI, this.bucket, 'o'].join('/');
+    this.uploadBaseURI = [uploadAPI, this.bucket, 'o'].join('/');
     this.authClient = new GoogleAuth(config);
-    this._checkBucket(bucketName);
+    this._checkBucket();
   }
 
   normalizeError(error: ClientError): HttpError {
@@ -131,7 +137,7 @@ export class GCStorage extends BaseStorage<GCSFile> {
   }
 
   async create(req: http.IncomingMessage, config: FileInit): Promise<GCSFile> {
-    const file = new GCSFile(config);
+    const file = new File(config) as GCSFile;
     file.name = this.namingFunction(file);
     await this.validate(file);
     try {
@@ -173,6 +179,7 @@ export class GCStorage extends BaseStorage<GCSFile> {
     if (isCompleted(file)) {
       file.uri = `${this.storageBaseURI}/${file.name}`;
       await this._onComplete(file);
+      return this.buildCompletedFile(file);
     }
     return file;
   }
@@ -190,6 +197,61 @@ export class GCStorage extends BaseStorage<GCSFile> {
     return [{ name } as GCSFile];
   }
 
+  async copy(name: string, dest: string): Promise<Record<string, string>> {
+    type CopyProgress = {
+      rewriteToken?: string;
+      kind: string;
+      objectSize: number;
+      totalBytesRewritten: number;
+      done: boolean;
+      resource: Record<string, any>;
+    };
+    const newPath = resolve(`/${this.bucket}/${name}`, encodeURI(dest));
+    const [, bucket, ...pathSegments] = newPath.split('/');
+    const filename = pathSegments.join('/');
+    const url = `${this.storageBaseURI}/${name}/rewriteTo/b/${bucket}/o/${filename}`;
+    let progress = {} as CopyProgress;
+    const opts = {
+      body: '',
+      headers: { 'Content-Type': 'application/json' },
+      method: 'POST' as const,
+      url
+    };
+    do {
+      opts.body = progress.rewriteToken
+        ? JSON.stringify({ rewriteToken: progress.rewriteToken })
+        : '';
+      progress = (await this.authClient.request<CopyProgress>(opts)).data;
+    } while (progress.rewriteToken);
+    return progress.resource;
+  }
+
+  async move(name: string, dest: string): Promise<Record<string, string>> {
+    const resource = await this.copy(name, dest);
+    const url = `${this.storageBaseURI}/${name}`;
+    await this.authClient.request({ method: 'DELETE' as const, url });
+    return resource;
+  }
+
+  async _get(name: string): Promise<Record<string, string>> {
+    const url = `${this.storageBaseURI}/${name}`;
+    return (await this.authClient.request<Record<string, string>>({ url })).data;
+  }
+
+  buildCompletedFile(file: GCSFile): GCSFile {
+    const completed = { ...file };
+    completed.lock = async lockFn => {
+      completed.lockedBy = lockFn;
+      return Promise.resolve(completed.lockedBy);
+    };
+    completed.get = () => this._get(file.name);
+    completed.delete = () => this.delete(file.name);
+    completed.copy = async (dest: string) => this.copy(file.name, dest);
+    completed.move = async (dest: string) => this.move(file.name, dest);
+
+    return completed;
+  }
+
   protected async _write(part: FilePart & GCSFile): Promise<number> {
     const { size, uri, body } = part;
     const contentRange = buildContentRange(part);
@@ -228,9 +290,9 @@ export class GCStorage extends BaseStorage<GCSFile> {
     return this.deleteMeta(file.name);
   };
 
-  private _checkBucket(bucketName: string): void {
+  private _checkBucket(): void {
     this.authClient
-      .request({ url: `${storageAPI}/${bucketName}` })
+      .request({ url: this.storageBaseURI })
       .then(() => (this.isReady = true))
       .catch((err: ClientError) => {
         // eslint-disable-next-line no-console

packages/s3/src/s3-storage.ts

@@ -2,8 +2,14 @@ import {
   AbortMultipartUploadCommand,
   CompleteMultipartUploadCommand,
   CompleteMultipartUploadOutput,
+  CopyObjectCommand,
+  CopyObjectCommandInput,
+  CopyObjectCommandOutput,
   CreateMultipartUploadCommand,
   CreateMultipartUploadRequest,
+  DeleteBucketCommand,
+  DeleteObjectCommandInput,
+  DeleteObjectCommandOutput,
   HeadBucketCommand,
   ListMultipartUploadsCommand,
   ListPartsCommand,
@@ -34,6 +40,7 @@ import {
 import * as http from 'http';
 import { AWSError } from './aws-error';
 import { S3MetaStorage, S3MetaStorageOptions } from './s3-meta-storage';
+import { resolve } from 'url';
 
 const BUCKET_NAME = 'node-uploadx';
 
@@ -42,8 +49,8 @@ export interface S3File extends File {
   UploadId?: string;
   uri?: string;
   lock: (lockFn: () => any) => Promise<any>;
-  move: (dest: any) => Promise<any>;
-  copy: (dest: any) => Promise<any>;
+  move: (dest: string) => Promise<Record<string, any>>;
+  copy: (dest: string) => Promise<Record<string, any>>;
   delete: () => Promise<any>;
 }
 
@@ -183,6 +190,7 @@ export class S3Storage extends BaseStorage<S3File> {
       const [completed] = await this._onComplete(file);
       delete file.Parts;
       file.uri = completed.Location;
+      return this.buildCompletedFile(file);
     }
     return file;
   }
@@ -197,6 +205,34 @@ export class S3Storage extends BaseStorage<S3File> {
     return [{ name } as S3File];
   }
 
+  async copy(name: string, dest: string): Promise<CopyObjectCommandOutput> {
+    const CopySource = encodeURI(`${this.bucket}/${name}`);
+    const newPath = resolve(`/${CopySource}`, encodeURI(dest));
+    const [, Bucket, ...pathSegments] = newPath.split('/');
+    const Key = pathSegments.join('/');
+    const params: CopyObjectCommandInput = { Bucket, Key, CopySource };
+    return this.client.send(new CopyObjectCommand(params));
+  }
+
+  async move(name: string, dest: string): Promise<DeleteObjectCommandOutput> {
+    await this.copy(name, dest);
+    const params: DeleteObjectCommandInput = { Bucket: this.bucket, Key: name };
+    return this.client.send(new DeleteBucketCommand(params));
+  }
+
+  buildCompletedFile(file: S3File): S3File {
+    const completed = { ...file };
+    completed.lock = async lockFn => {
+      completed.lockedBy = lockFn;
+      return Promise.resolve(completed.lockedBy);
+    };
+    completed.delete = () => this.delete(file.name);
+    completed.copy = async (dest: string) => this.copy(file.name, dest);
+    completed.move = async (dest: string) => this.move(file.name, dest);
+
+    return completed;
+  }
+
   protected _onComplete = (file: S3File): Promise<[CompleteMultipartUploadOutput, any]> => {
     return Promise.all([this._complete(file), this.deleteMeta(file.name)]);
   };

test/gcs-storage.spec.ts

@@ -30,7 +30,7 @@ describe('GCStorage', () => {
   let storage: GCStorage;
   let file: GCSFile;
   const uri = 'http://api.com?upload_id=123456789';
-  const _fileResponse = (): { data: GCSFile } => ({ data: { ...testfile, uri } });
+  const _fileResponse = (): { data: GCSFile } => ({ data: { ...testfile, uri } as GCSFile });
   const _createResponse = (): any => ({ headers: { location: uri } });
   const req = { headers: { origin: 'http://api.com' } } as IncomingMessage;