Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix(storage): add copy options for addMetadata #975

Merged
merged 4 commits into from
Jul 16, 2024
Merged

fix(storage): add copy options for addMetadata #975

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
cf3730b
fix(storage): add copy options for addMetadata
maxakuru Jul 15, 2024
6f4afe4
chore: fix tests
maxakuru Jul 15, 2024
5fb2264
Apply suggestions from code review
maxakuru Jul 16, 2024
9916c3a
chore: tests
maxakuru Jul 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 37 additions & 15 deletions packages/helix-shared-storage/src/storage.d.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,32 @@

import { S3Client } from "@aws-sdk/client-s3";


export interface ObjectInfo {
key: string;
/** the path to the object, w/o the prefix */
path: string;
lastModified: string;
contentLength: number;
contentType: string;
}

/**
* @returns {boolean} {@code true} if the object is accepted
*/
export type ObjectFilter = (info: ObjectInfo) => boolean;

export interface CopyOptions {
/** metadata to merge with existing metadata */
addMetadata?: Record<string, unknown>;
}

export declare interface Bucket {
get client():S3Client;
get client(): S3Client;

get bucket():string;
get bucket(): string;

get log():Console;
get log(): Console;

get(key: string, meta?: object): Promise<Buffer | null>;

Expand Down Expand Up @@ -51,7 +71,7 @@ export declare interface Bucket {
* @param {boolean} [compress = true]
* @returns result obtained from S3
*/
put(path: string, body: Buffer, contentType?: string, meta?: object, compress?: bool): Promise<object>;
put(path: string, body: Buffer, contentType?: string, meta?: object, compress?: boolean): Promise<object>;

/**
* Updates the metadata
Expand All @@ -67,9 +87,10 @@ export declare interface Bucket {
*
* @param {string} src source key
* @param {string} dst destination key
* @param {CopyOptions} [opts]
* @returns result obtained from S3
*/
copy(src: string, dst: string): Promise<void>;
copy(src: string, dst: string, opts?: CopyOptions): Promise<void>;

/**
* Remove object(s)
Expand All @@ -93,50 +114,51 @@ export declare interface Bucket {
* @param {string} src Source prefix
* @param {string} dst Destination prefix
* @param {ObjectFilter} filter Filter function
* @param {CopyOptions} [opts]
* @returns {Promise<*[]>}
*/
copyDeep(src: string, dst: string, filter?: function): Promise<object[]>;
copyDeep(src: string, dst: string, filter?: ObjectFilter, opts?: CopyOptions): Promise<object[]>;

rmdir(src: string): Promise<void>;
}

/**
* The Helix Storage provides a factory for simplified bucket operations to S3 and R2
*/
export class HelixStorage {
static fromContext(context:AdminContext):HelixStorage;
export declare class HelixStorage {
static fromContext(context: AdminContext): HelixStorage;

s3():S3Client;
s3(): S3Client;

/**
* creates a bucket instance that allows to perform storage related operations.
* @param bucketId
* @returns {Bucket}
*/
bucket(bucketId:string):Bucket;;
bucket(bucketId: string): Bucket;;

/**
* @returns {Bucket}
*/
contentBus():Bucket;
contentBus(): Bucket;

/**
* @returns {Bucket}
*/
codeBus():Bucket;
codeBus(): Bucket;

/**
* @returns {Bucket}
*/
mediaBus():Bucket;
mediaBus(): Bucket;

/**
* @returns {Bucket}
*/
configBus():Bucket;
configBus(): Bucket;

/**
* Close this storage. Destroys the S3 client used.
*/
close()
close(): void;
}
41 changes: 23 additions & 18 deletions packages/helix-shared-storage/src/storage.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,21 +35,11 @@ const gunzip = promisify(zlib.gunzip);

/**
* @typedef {import('@aws-sdk/client-s3').CommandInput} CommandInput
*/

/**
* @typedef ObjectInfo
* @property {string} key
* @property {string} path the path to the object, w/o the prefix
* @property {string} lastModified
* @property {number} contentLength
* @property {string} contentType
*/

/**
* @callback ObjectFilter
* @param {ObjectInfo} info of the object to filter
* @returns {boolean} {@code true} if the object is accepted
* @typedef {import('./storage.d').Bucket} BucketType
* @typedef {import('./storage.d').HelixStorage} HelixStorageType
* @typedef {import('./storage.d').ObjectInfo} ObjectInfo
* @typedef {import('./storage.d').ObjectFilter} ObjectFilter
* @typedef {import('./storage.d').CopyOptions} CopyOptions
*/

/**
Expand Down Expand Up @@ -92,6 +82,7 @@ function sanitizeKey(keyOrPath) {

/**
* Bucket class
* @implements {BucketType}
*/
class Bucket {
constructor(opts) {
Expand Down Expand Up @@ -308,16 +299,23 @@ class Bucket {
*
* @param {string} src source key
* @param {string} dst destination key
* @param {CopyOptions} [opts]
* @returns result obtained from S3
*/
async copy(src, dst) {
async copy(src, dst, opts = {}) {
const key = sanitizeKey(src);
const input = {
Bucket: this.bucket,
CopySource: `${this.bucket}/${sanitizeKey(src)}`,
CopySource: `${this.bucket}/${key}`,
Key: sanitizeKey(dst),
};

try {
if (opts.addMetadata) {
const meta = await this.metadata(key);
input.Metadata = { ...meta, ...opts.addMetadata };
maxakuru marked this conversation as resolved.
Show resolved Hide resolved
input.MetadataDirective = 'REPLACE';
}
// write to s3 and r2 (mirror) in parallel
await this.sendToS3andR2(CopyObjectCommand, input);
this.log.info(`object copied from ${input.CopySource} to: ${input.Bucket}/${input.Key}`);
Expand Down Expand Up @@ -432,9 +430,10 @@ class Bucket {
* @param {string} src Source prefix
* @param {string} dst Destination prefix
* @param {ObjectFilter} filter Filter function
* @param {CopyOptions} [opts={}]
* @returns {Promise<*[]>}
*/
async copyDeep(src, dst, filter = () => true) {
async copyDeep(src, dst, filter = () => true, opts = {}) {
const { log } = this;
const tasks = [];
const Prefix = sanitizeKey(src);
Expand Down Expand Up @@ -465,6 +464,11 @@ class Bucket {
Key: task.dst,
};
try {
if (opts.addMetadata) {
const meta = await this.metadata(task.src);
input.Metadata = { ...meta, ...opts.addMetadata };
maxakuru marked this conversation as resolved.
Show resolved Hide resolved
input.MetadataDirective = 'REPLACE';
}
// write to s3 and r2 (mirror) in parallel
await this.sendToS3andR2(CopyObjectCommand, input);
changes.push(task);
Expand Down Expand Up @@ -510,6 +514,7 @@ class Bucket {

/**
* The Helix Storage provides a factory for simplified bucket operations to S3 and R2
* @implements {HelixStorageType}
*/
export class HelixStorage {
static fromContext(context) {
Expand Down
159 changes: 159 additions & 0 deletions packages/helix-shared-storage/test/storage.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -476,6 +476,111 @@ describe('Storage test', () => {
assert.deepEqual(puts.r2, expectedPuts);
});

it('can copy objects and add metadata', async () => {
const listReply = JSON.parse(await fs.readFile(path.resolve(__testdir, 'fixtures', 'list-reply-copy.json'), 'utf-8'));
const puts = { s3: [], r2: [] };
const putsHeaders = { s3: [], r2: [] };
const heads = [];
nock('https://helix-code-bus.s3.fake.amazonaws.com')
.get('/?list-type=2&prefix=owner%2Frepo%2Fref%2F')
.reply(200, listReply[0])
.get('/?continuation-token=1%2Fs4dr7BSKNScrN4njX9%2BCpBNimYkuEzMWg3niTSAPMdculBmycyUPM6kv0xi46j4hdc1lFPkE%2FICI8TxG%2BVNV9Hh91Ou0hqeBYzqTRzSBSs%3D&list-type=2&prefix=owner%2Frepo%2Fref%2F')
.reply(200, listReply[1])
.head(/.*/)
.times(10)
.reply((uri) => {
heads.push(uri);
// reject first uri
if (puts.s3.length === 1) {
return [404];
}
return [200, undefined, {
'x-amz-meta-x-dont-overwrite': 'foo',
'x-amz-meta-x-last-modified-by': 'anonymous',
}];
})
.put(/.*/)
.times(10)
.reply(function f(uri) {
puts.s3.push(uri);
putsHeaders.s3.push({
'x-amz-metadata-directive': this.req.headers['x-amz-metadata-directive'],
'x-amz-meta-x-dont-overwrite': this.req.headers['x-amz-meta-x-dont-overwrite'],
'x-amz-meta-x-last-modified-by': this.req.headers['x-amz-meta-x-last-modified-by'],
});
// reject first uri
if (puts.s3.length === 1) {
return [404];
}
return [200, '<?xml version="1.0" encoding="UTF-8"?>\n<CopyObjectResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><LastModified>2021-05-05T08:37:23.000Z</LastModified><ETag>&quot;f278c0035a9b4398629613a33abe6451&quot;</ETag></CopyObjectResult>'];
});
nock(`https://helix-code-bus.${CLOUDFLARE_ACCOUNT_ID}.r2.cloudflarestorage.com`)
.put(/.*/)
.times(10)
.reply(function f(uri) {
puts.r2.push(uri);
putsHeaders.r2.push({
'x-amz-metadata-directive': this.req.headers['x-amz-metadata-directive'],
'x-amz-meta-x-dont-overwrite': this.req.headers['x-amz-meta-x-dont-overwrite'],
'x-amz-meta-x-last-modified-by': this.req.headers['x-amz-meta-x-last-modified-by'],
});
// reject first uri
if (puts.s3.length === 1) {
return [404];
}
return [200, '<?xml version="1.0" encoding="UTF-8"?>\n<CopyObjectResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><LastModified>2021-05-05T08:37:23.000Z</LastModified><ETag>&quot;f278c0035a9b4398629613a33abe6451&quot;</ETag></CopyObjectResult>'];
});

const bus = storage.codeBus();
await bus.copyDeep('/owner/repo/ref/', '/bar/', undefined, { addMetadata: { 'x-last-modified-by': 'foo@example.com' } });

puts.s3.sort();
puts.r2.sort();
heads.sort();

assert.strictEqual(putsHeaders.s3.length, 10);
assert.strictEqual(putsHeaders.r2.length, 10);

Object.values(putsHeaders).forEach((s3r2) => {
s3r2.forEach((headers) => {
assert.deepEqual(headers, {
'x-amz-meta-x-dont-overwrite': 'foo',
'x-amz-meta-x-last-modified-by': 'foo@example.com',
'x-amz-metadata-directive': 'REPLACE',
});
});
});

const expectedHeads = [
'/owner/repo/ref/.circleci/config.yml',
'/owner/repo/ref/.gitignore',
'/owner/repo/ref/.vscode/launch.json',
'/owner/repo/ref/.vscode/settings.json',
'/owner/repo/ref/README.md',
'/owner/repo/ref/helix_logo.png',
'/owner/repo/ref/htdocs/favicon.ico',
'/owner/repo/ref/htdocs/style.css',
'/owner/repo/ref/index.md',
'/owner/repo/ref/src/html.pre.js',
];
assert.deepEqual(heads, expectedHeads);

const expectedPuts = [
'/bar/.circleci/config.yml?x-id=CopyObject',
'/bar/.gitignore?x-id=CopyObject',
'/bar/.vscode/launch.json?x-id=CopyObject',
'/bar/.vscode/settings.json?x-id=CopyObject',
'/bar/README.md?x-id=CopyObject',
'/bar/helix_logo.png?x-id=CopyObject',
'/bar/htdocs/favicon.ico?x-id=CopyObject',
'/bar/htdocs/style.css?x-id=CopyObject',
'/bar/index.md?x-id=CopyObject',
'/bar/src/html.pre.js?x-id=CopyObject',
];
assert.deepEqual(puts.s3, expectedPuts);
assert.deepEqual(puts.r2, expectedPuts);
});

it('can copy object (non deep)', async () => {
const puts = { s3: [], r2: [] };
nock('https://helix-code-bus.s3.fake.amazonaws.com')
Expand Down Expand Up @@ -503,6 +608,60 @@ describe('Storage test', () => {
assert.deepEqual(puts.r2, expectedPuts);
});

it('can copy object, and add metadata (non deep)', async () => {
const puts = { s3: [], r2: [] };
const putsHeaders = { s3: undefined, r2: undefined };
nock('https://helix-code-bus.s3.fake.amazonaws.com')
.head('/owner/repo/ref/foo.md')
.reply(200, undefined, {
'x-amz-meta-x-dont-overwrite': 'foo',
'x-amz-meta-x-last-modified-by': 'anonymous',
})
.put('/owner/repo/ref/foo/bar.md?x-id=CopyObject')
.reply(function f(uri) {
putsHeaders.s3 = {
'x-amz-metadata-directive': this.req.headers['x-amz-metadata-directive'],
'x-amz-meta-x-dont-overwrite': this.req.headers['x-amz-meta-x-dont-overwrite'],
'x-amz-meta-x-last-modified-by': this.req.headers['x-amz-meta-x-last-modified-by'],
};
puts.s3.push(uri);
return [200, '<?xml version="1.0" encoding="UTF-8"?>\n<CopyObjectResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><LastModified>2021-05-05T08:37:23.000Z</LastModified><ETag>&quot;f278c0035a9b4398629613a33abe6451&quot;</ETag></CopyObjectResult>'];
});
nock(`https://helix-code-bus.${CLOUDFLARE_ACCOUNT_ID}.r2.cloudflarestorage.com`)
.put('/owner/repo/ref/foo/bar.md?x-id=CopyObject')
.reply(function f(uri) {
putsHeaders.r2 = {
'x-amz-metadata-directive': this.req.headers['x-amz-metadata-directive'],
'x-amz-meta-x-dont-overwrite': this.req.headers['x-amz-meta-x-dont-overwrite'],
'x-amz-meta-x-last-modified-by': this.req.headers['x-amz-meta-x-last-modified-by'],
};
puts.r2.push(uri);
return [200, '<?xml version="1.0" encoding="UTF-8"?>\n<CopyObjectResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><LastModified>2021-05-05T08:37:23.000Z</LastModified><ETag>&quot;f278c0035a9b4398629613a33abe6451&quot;</ETag></CopyObjectResult>'];
});

const bus = storage.codeBus();
await bus.copy('/owner/repo/ref/foo.md', '/owner/repo/ref/foo/bar.md', { addMetadata: { 'x-last-modified-by': 'foo@example.com' } });

puts.s3.sort();
puts.r2.sort();
const expectedPuts = [
'/owner/repo/ref/foo/bar.md?x-id=CopyObject',
];
assert.deepEqual(puts.s3, expectedPuts);
assert.deepEqual(puts.r2, expectedPuts);

assert.deepEqual(putsHeaders.s3, {
'x-amz-metadata-directive': 'REPLACE',
'x-amz-meta-x-dont-overwrite': 'foo',
'x-amz-meta-x-last-modified-by': 'foo@example.com',
});
assert.deepEqual(putsHeaders.r2, {
'x-amz-metadata-directive': 'REPLACE',
'x-amz-meta-x-dont-overwrite': 'foo',
'x-amz-meta-x-last-modified-by': 'foo@example.com',
});
});

it('can copy object can fail (non deep)', async () => {
nock('https://helix-code-bus.s3.fake.amazonaws.com')
.put('/owner/repo/ref/foo/bar.md?x-id=CopyObject')
Expand Down
Loading