Skip to content

Commit

Permalink
fix: implement cloud file operations
Browse files Browse the repository at this point in the history
  • Loading branch information
@kukhariev
kukhariev committed Sep 3, 2021
1 parent 22c2955 commit f4356d9
Show file tree
Hide file tree
Showing 4 changed files with 144 additions and 20 deletions.
8 changes: 4 additions & 4 deletions examples/express-gcs.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@ const app = express();

const storage = new GCStorage({ maxUploadSize: '1GB' });

storage.onComplete = ({ uri, id }) => {
console.log(`File upload complete, storage path: ${uri}`);
// send gcs link to client
return { id, link: uri };
storage.onComplete = async file => {
const info = await file.get().catch(console.error);
console.log(info);
return file.move(file.originalName).catch(console.error);
};

app.use('/files', uploadx({ storage }));
Expand Down
80 changes: 71 additions & 9 deletions packages/gcs/src/gcs-storage.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import * as http from 'http';
import request from 'node-fetch';
import { authScopes, BUCKET_NAME, storageAPI, uploadAPI } from './constants';
import { GCSMetaStorage, GCSMetaStorageOptions } from './gcs-meta-storage';
import { resolve } from 'url';

export interface ClientError extends Error {
code: string;
Expand Down Expand Up @@ -72,9 +73,13 @@ export interface GCStorageOptions extends BaseStorageOptions<GCSFile>, GoogleAut
metaStorageConfig?: LocalMetaStorageOptions | GCSMetaStorageOptions;
}

export class GCSFile extends File {
export interface GCSFile extends File {
GCSUploadURI?: string;
uri = '';
uri: string;
move: (dest: string) => Promise<Record<string, string>>;
copy: (dest: string) => Promise<Record<string, string>>;
get: () => Promise<Record<string, string>>;
delete: () => Promise<any>;
}

/**
Expand All @@ -95,6 +100,7 @@ export class GCStorage extends BaseStorage<GCSFile> {
storageBaseURI: string;
uploadBaseURI: string;
meta: MetaStorage<GCSFile>;
private readonly bucket: string;

constructor(public config: GCStorageOptions = {}) {
super(config);
Expand All @@ -109,11 +115,11 @@ export class GCStorage extends BaseStorage<GCSFile> {
}
config.scopes ||= authScopes;
config.keyFile ||= process.env.GCS_KEYFILE;
const bucketName = config.bucket || process.env.GCS_BUCKET || BUCKET_NAME;
this.storageBaseURI = [storageAPI, bucketName, 'o'].join('/');
this.uploadBaseURI = [uploadAPI, bucketName, 'o'].join('/');
this.bucket = config.bucket || process.env.GCS_BUCKET || BUCKET_NAME;
this.storageBaseURI = [storageAPI, this.bucket, 'o'].join('/');
this.uploadBaseURI = [uploadAPI, this.bucket, 'o'].join('/');
this.authClient = new GoogleAuth(config);
this._checkBucket(bucketName);
this._checkBucket();
}

normalizeError(error: ClientError): HttpError {
Expand All @@ -131,7 +137,7 @@ export class GCStorage extends BaseStorage<GCSFile> {
}

async create(req: http.IncomingMessage, config: FileInit): Promise<GCSFile> {
const file = new GCSFile(config);
const file = new File(config) as GCSFile;
file.name = this.namingFunction(file);
await this.validate(file);
try {
Expand Down Expand Up @@ -173,6 +179,7 @@ export class GCStorage extends BaseStorage<GCSFile> {
if (isCompleted(file)) {
file.uri = `${this.storageBaseURI}/${file.name}`;
await this._onComplete(file);
return this.buildCompletedFile(file);
}
return file;
}
Expand All @@ -190,6 +197,61 @@ export class GCStorage extends BaseStorage<GCSFile> {
return [{ name } as GCSFile];
}

async copy(name: string, dest: string): Promise<Record<string, string>> {
type CopyProgress = {
rewriteToken?: string;
kind: string;
objectSize: number;
totalBytesRewritten: number;
done: boolean;
resource: Record<string, any>;
};
const newPath = resolve(`/${this.bucket}/${name}`, encodeURI(dest));
const [, bucket, ...pathSegments] = newPath.split('/');
const filename = pathSegments.join('/');
const url = `${this.storageBaseURI}/${name}/rewriteTo/b/${bucket}/o/${filename}`;
let progress = {} as CopyProgress;
const opts = {
body: '',
headers: { 'Content-Type': 'application/json' },
method: 'POST' as const,
url
};
do {
opts.body = progress.rewriteToken
? JSON.stringify({ rewriteToken: progress.rewriteToken })
: '';
progress = (await this.authClient.request<CopyProgress>(opts)).data;
} while (progress.rewriteToken);
return progress.resource;
}

async move(name: string, dest: string): Promise<Record<string, string>> {
const resource = await this.copy(name, dest);
const url = `${this.storageBaseURI}/${name}`;
await this.authClient.request({ method: 'DELETE' as const, url });
return resource;
}

async _get(name: string): Promise<Record<string, string>> {
const url = `${this.storageBaseURI}/${name}`;
return (await this.authClient.request<Record<string, string>>({ url })).data;
}

buildCompletedFile(file: GCSFile): GCSFile {
const completed = { ...file };
completed.lock = async lockFn => {
completed.lockedBy = lockFn;
return Promise.resolve(completed.lockedBy);
};
completed.get = () => this._get(file.name);
completed.delete = () => this.delete(file.name);
completed.copy = async (dest: string) => this.copy(file.name, dest);
completed.move = async (dest: string) => this.move(file.name, dest);

return completed;
}

protected async _write(part: FilePart & GCSFile): Promise<number> {
const { size, uri, body } = part;
const contentRange = buildContentRange(part);
Expand Down Expand Up @@ -228,9 +290,9 @@ export class GCStorage extends BaseStorage<GCSFile> {
return this.deleteMeta(file.name);
};

private _checkBucket(bucketName: string): void {
private _checkBucket(): void {
this.authClient
.request({ url: `${storageAPI}/${bucketName}` })
.request({ url: this.storageBaseURI })
.then(() => (this.isReady = true))
.catch((err: ClientError) => {
// eslint-disable-next-line no-console
Expand Down
42 changes: 39 additions & 3 deletions packages/s3/src/s3-storage.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,13 @@ import {
AbortMultipartUploadCommand,
CompleteMultipartUploadCommand,
CompleteMultipartUploadOutput,
CopyObjectCommand,
CopyObjectCommandInput,
CopyObjectCommandOutput,
CreateMultipartUploadCommand,
CreateMultipartUploadRequest,
DeleteObjectCommand,
DeleteObjectCommandInput,
HeadBucketCommand,
ListMultipartUploadsCommand,
ListPartsCommand,
Expand Down Expand Up @@ -34,16 +39,17 @@ import {
import * as http from 'http';
import { AWSError } from './aws-error';
import { S3MetaStorage, S3MetaStorageOptions } from './s3-meta-storage';
import { resolve } from 'url';

const BUCKET_NAME = 'node-uploadx';

export interface S3File extends File {
Parts?: Part[];
UploadId?: string;
uri?: string;
lock: (lockFn: () => any) => Promise<any>;
move: (dest: any) => Promise<any>;
copy: (dest: any) => Promise<any>;
move: (dest: string) => Promise<Record<string, any>>;
copy: (dest: string) => Promise<Record<string, any>>;
get: () => Promise<Record<string, any>>;
delete: () => Promise<any>;
}

Expand Down Expand Up @@ -183,6 +189,7 @@ export class S3Storage extends BaseStorage<S3File> {
const [completed] = await this._onComplete(file);
delete file.Parts;
file.uri = completed.Location;
return this.buildCompletedFile(file);
}
return file;
}
Expand All @@ -197,6 +204,35 @@ export class S3Storage extends BaseStorage<S3File> {
return [{ name } as S3File];
}

async copy(name: string, dest: string): Promise<CopyObjectCommandOutput> {
const CopySource = `${this.bucket}/${name}`;
const newPath = decodeURI(resolve(`/${CopySource}`, dest)); // path.resolve?
const [, Bucket, ...pathSegments] = newPath.split('/');
const Key = pathSegments.join('/');
const params: CopyObjectCommandInput = { Bucket, Key, CopySource };
return this.client.send(new CopyObjectCommand(params));
}

async move(name: string, dest: string): Promise<CopyObjectCommandOutput> {
const copyOut = await this.copy(name, dest);
const params: DeleteObjectCommandInput = { Bucket: this.bucket, Key: name };
await this.client.send(new DeleteObjectCommand(params));
return copyOut;
}

buildCompletedFile(file: S3File): S3File {
const completed = { ...file };
completed.lock = async lockFn => {
completed.lockedBy = lockFn;
return Promise.resolve(completed.lockedBy);
};
completed.delete = () => this.delete(file.name);
completed.copy = async (dest: string) => this.copy(file.name, dest);
completed.move = async (dest: string) => this.move(file.name, dest);

return completed;
}

protected _onComplete = (file: S3File): Promise<[CompleteMultipartUploadOutput, any]> => {
return Promise.all([this._complete(file), this.deleteMeta(file.name)]);
};
Expand Down
34 changes: 30 additions & 4 deletions test/gcs-storage.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,13 @@ describe('GCStorage', () => {
let storage: GCStorage;
let file: GCSFile;
const uri = 'http://api.com?upload_id=123456789';
const _fileResponse = (): { data: GCSFile } => ({ data: { ...testfile, uri } });
const _fileResponse = (): { data: GCSFile } => ({ data: { ...testfile, uri } as GCSFile });
const _createResponse = (): any => ({ headers: { location: uri } });
const req = { headers: { origin: 'http://api.com' } } as IncomingMessage;

beforeEach(async () => {
mockAuthRequest.mockResolvedValueOnce({ bucket: 'ok' });
storage = new GCStorage({ ...(storageOptions as GCStorageOptions) });
storage = new GCStorage({ ...(storageOptions as GCStorageOptions), bucket: 'test' });
file = _fileResponse().data;
});

Expand Down Expand Up @@ -79,13 +79,13 @@ describe('GCStorage', () => {
});
});

describe('.get()', () => {
describe('.list()', () => {
it('should return all user files', async () => {
const list = {
data: { items: [{ name: metafile }] }
};
mockAuthRequest.mockResolvedValue(list);
const { items } = await storage.get(testfile.userId);
const { items } = await storage.list(testfile.userId);
expect(items).toEqual(expect.any(Array));
expect(items).toHaveLength(1);
expect(items[0]).toMatchObject({ name: filename });
Expand Down Expand Up @@ -144,6 +144,32 @@ describe('GCStorage', () => {
expect(deleted.status).toBe('deleted');
});
});

describe('.copy()', () => {
it('relative', async () => {
mockAuthRequest.mockResolvedValue({ data: { done: true } });
await storage.copy(filename, 'files/новое имя.txt');
expect(mockAuthRequest).toHaveBeenCalledWith({
body: '',
headers: { 'Content-Type': 'application/json' },
method: 'POST',
url:
'https://storage.googleapis.com/storage/v1/b/test/o/userId/testfile.mp4/rewriteTo/b/test/o/userId/' +
'files/%D0%BD%D0%BE%D0%B2%D0%BE%D0%B5%20%D0%B8%D0%BC%D1%8F.txt'
});
});

it('absolute', async () => {
mockAuthRequest.mockResolvedValue({ data: { done: true } });
await storage.copy(filename, '/new/name.txt');
expect(mockAuthRequest).toHaveBeenCalledWith({
body: '',
headers: { 'Content-Type': 'application/json' },
method: 'POST',
url: 'https://storage.googleapis.com/storage/v1/b/test/o/userId/testfile.mp4/rewriteTo/b/new/o/name.txt'
});
});
});
});

describe('Range utils', () => {
Expand Down

0 comments on commit f4356d9

Please sign in to comment.