Skip to content

Commit

Permalink
fix: implement cloud file operations
Browse files Browse the repository at this point in the history
  • Loading branch information
kukhariev committed Sep 2, 2021
1 parent 22c2955 commit 4c9848d
Show file tree
Hide file tree
Showing 3 changed files with 90 additions and 6 deletions.
55 changes: 52 additions & 3 deletions packages/gcs/src/gcs-storage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -72,9 +72,12 @@ export interface GCStorageOptions extends BaseStorageOptions<GCSFile>, GoogleAut
metaStorageConfig?: LocalMetaStorageOptions | GCSMetaStorageOptions;
}

export class GCSFile extends File {
export interface GCSFile extends File {
GCSUploadURI?: string;
uri = '';
uri: string;
move: (dest: string) => Promise<any>;
copy: (dest: string) => Promise<any>;
delete: () => Promise<any>;
}

/**
Expand Down Expand Up @@ -131,7 +134,7 @@ export class GCStorage extends BaseStorage<GCSFile> {
}

async create(req: http.IncomingMessage, config: FileInit): Promise<GCSFile> {
const file = new GCSFile(config);
const file = new File(config) as GCSFile;
file.name = this.namingFunction(file);
await this.validate(file);
try {
Expand Down Expand Up @@ -173,6 +176,7 @@ export class GCStorage extends BaseStorage<GCSFile> {
if (isCompleted(file)) {
file.uri = `${this.storageBaseURI}/${file.name}`;
await this._onComplete(file);
return this.buildCompletedFile(file);
}
return file;
}
Expand All @@ -190,6 +194,51 @@ export class GCStorage extends BaseStorage<GCSFile> {
return [{ name } as GCSFile];
}

async copy(name: string, dest: string): Promise<any> {
type CopyProgress = {
rewriteToken?: string | number;
kind: string;
objectSize: number;
totalBytesRewritten: number;
done: boolean;
resource?: Record<string, any>;
};
const url = `${this.storageBaseURI}/${name}/rewriteTo/b/${encodeURI(dest).replace('/', '/o/')}`;

let progress = {} as CopyProgress;
do {
const opts = {
body: '',
headers: { 'Content-Type': 'application/json' },
method: 'POST' as const,
url
};
progress.rewriteToken &&
(opts.body = JSON.stringify({ rewriteToken: progress.rewriteToken }));
progress = (await this.authClient.request<CopyProgress>(opts)).data;
} while (progress.rewriteToken);
return progress.resource;
}

async move(name: string, dest: string): Promise<any> {
await this.copy(name, dest);
const url = `${this.storageBaseURI}/${name}`;
await this.authClient.request({ method: 'DELETE' as const, url });
}

buildCompletedFile(file: GCSFile): GCSFile {
const completed = { ...file };
completed.lock = async lockFn => {
completed.lockedBy = lockFn;
return Promise.resolve(completed.lockedBy);
};
completed.delete = () => this.delete(file.name);
completed.copy = async (dest: string) => this.copy(file.name, dest);
completed.move = async (dest: string) => this.move(file.name, dest);

return completed;
}

protected async _write(part: FilePart & GCSFile): Promise<number> {
const { size, uri, body } = part;
const contentRange = buildContentRange(part);
Expand Down
39 changes: 37 additions & 2 deletions packages/s3/src/s3-storage.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,14 @@ import {
AbortMultipartUploadCommand,
CompleteMultipartUploadCommand,
CompleteMultipartUploadOutput,
CopyObjectCommand,
CopyObjectCommandInput,
CopyObjectCommandOutput,
CreateMultipartUploadCommand,
CreateMultipartUploadRequest,
DeleteBucketCommand,
DeleteObjectCommandInput,
DeleteObjectCommandOutput,
HeadBucketCommand,
ListMultipartUploadsCommand,
ListPartsCommand,
Expand Down Expand Up @@ -42,8 +48,8 @@ export interface S3File extends File {
UploadId?: string;
uri?: string;
lock: (lockFn: () => any) => Promise<any>;
move: (dest: any) => Promise<any>;
copy: (dest: any) => Promise<any>;
move: (dest: string) => Promise<any>;
copy: (dest: string) => Promise<any>;
delete: () => Promise<any>;
}

Expand Down Expand Up @@ -183,6 +189,7 @@ export class S3Storage extends BaseStorage<S3File> {
const [completed] = await this._onComplete(file);
delete file.Parts;
file.uri = completed.Location;
return this.buildCompletedFile(file);
}
return file;
}
Expand All @@ -197,6 +204,34 @@ export class S3Storage extends BaseStorage<S3File> {
return [{ name } as S3File];
}

async copy(name: string, dest: string): Promise<CopyObjectCommandOutput> {
const params: CopyObjectCommandInput = {
Bucket: dest.split('/')[0] || this.bucket,
Key: dest,
CopySource: encodeURI(`${this.bucket}/${name}`)
};
return this.client.send(new CopyObjectCommand(params));
}

async move(name: string, dest: string): Promise<DeleteObjectCommandOutput> {
await this.copy(name, dest);
const params: DeleteObjectCommandInput = { Bucket: this.bucket, Key: name };
return this.client.send(new DeleteBucketCommand(params));
}

buildCompletedFile(file: S3File): S3File {
const completed = { ...file };
completed.lock = async lockFn => {
completed.lockedBy = lockFn;
return Promise.resolve(completed.lockedBy);
};
completed.delete = () => this.delete(file.name);
completed.copy = async (dest: string) => this.copy(file.name, dest);
completed.move = async (dest: string) => this.move(file.name, dest);

return completed;
}

protected _onComplete = (file: S3File): Promise<[CompleteMultipartUploadOutput, any]> => {
return Promise.all([this._complete(file), this.deleteMeta(file.name)]);
};
Expand Down
2 changes: 1 addition & 1 deletion test/gcs-storage.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ describe('GCStorage', () => {
let storage: GCStorage;
let file: GCSFile;
const uri = 'http://api.com?upload_id=123456789';
const _fileResponse = (): { data: GCSFile } => ({ data: { ...testfile, uri } });
const _fileResponse = (): { data: GCSFile } => ({ data: { ...testfile, uri } as GCSFile });
const _createResponse = (): any => ({ headers: { location: uri } });
const req = { headers: { origin: 'http://api.com' } } as IncomingMessage;

Expand Down

0 comments on commit 4c9848d

Please sign in to comment.