From e5ecce7857717fc84f5f4f0f1951ec8f06e50dba Mon Sep 17 00:00:00 2001 From: George Fu Date: Fri, 26 Sep 2025 13:56:02 -0400 Subject: [PATCH 1/4] test(lib-storage): example of how to mock requests for MPU --- lib/lib-storage/package.json | 4 +- .../src/lib-storage-example.integ.spec.ts | 280 ++++++++++++++++++ lib/lib-storage/vitest.config.integ.mts | 8 + 3 files changed, 291 insertions(+), 1 deletion(-) create mode 100644 lib/lib-storage/src/lib-storage-example.integ.spec.ts create mode 100644 lib/lib-storage/vitest.config.integ.mts diff --git a/lib/lib-storage/package.json b/lib/lib-storage/package.json index 30998616e9ecf..9db1f62b9f7ba 100644 --- a/lib/lib-storage/package.json +++ b/lib/lib-storage/package.json @@ -15,8 +15,10 @@ "clean": "rimraf ./dist-* && rimraf *.tsbuildinfo", "extract:docs": "api-extractor run --local", "test": "yarn g:vitest run", - "test:e2e": "yarn g:vitest run -c vitest.config.e2e.mts --mode development", "test:watch": "yarn g:vitest watch", + "test:integration": "yarn g:vitest run -c vitest.config.integ.mts", + "test:integration:watch": "yarn g:vitest watch -c vitest.config.integ.mts", + "test:e2e": "yarn g:vitest run -c vitest.config.e2e.mts --mode development", "test:e2e:watch": "yarn g:vitest watch -c vitest.config.e2e.mts" }, "engines": { diff --git a/lib/lib-storage/src/lib-storage-example.integ.spec.ts b/lib/lib-storage/src/lib-storage-example.integ.spec.ts new file mode 100644 index 0000000000000..f27ad6bdc938c --- /dev/null +++ b/lib/lib-storage/src/lib-storage-example.integ.spec.ts @@ -0,0 +1,280 @@ +import { getE2eTestResources, requireRequestsFrom } from "@aws-sdk/aws-util-test/src"; +import { S3 } from "@aws-sdk/client-s3"; +import { Upload } from "@aws-sdk/lib-storage"; +import { getHttpDebugLogPlugin } from "@aws-sdk/middleware-http-debug-log/src"; +import { HttpResponse } from "@smithy/protocol-http"; +import { randomBytes } from "crypto"; +import { Readable } from "node:stream"; +import { describe, expect, test as it } from "vitest"; + +describe("lib storage integration test", () => { + const example = async () => { + const e2eTestResourcesEnv = await getE2eTestResources(); + Object.assign(process.env, e2eTestResourcesEnv); + + const region = process?.env?.AWS_SMOKE_TEST_REGION as string; + const Bucket = process?.env?.AWS_SMOKE_TEST_BUCKET as string; + const data = randomBytes(6 * 1024 * 1024); + + const s3 = new S3({ + region, + }); + const Key = `MPU-${Date.now()}`; + + const client = new S3({ + region, + }); + // This will print out all requests and responses so you can use + // them in an integration mock later. + client.middlewareStack.use( + getHttpDebugLogPlugin({ + request: { + url: true, + command: true, + method: true, + }, + response: { + statusCode: true, + headers: true, + body: true, + formatBody: true, + }, + }) + ); + + await new Upload({ + client, + params: { + Bucket, + Key, + Body: data, + }, + }).done(); + + await s3.deleteObject({ + Bucket, + Key, + }); + }; + + it("example of how to write an integration test that includes responses", async () => { + const client = new S3({ + region: "us-west-2", + credentials: { + accessKeyId: "INTEG", + secretAccessKey: "INTEG", + }, + }); + + const commandOutputs: Record = {}; + + client.middlewareStack.add((next, context) => async (args) => { + const r = await next(args); + commandOutputs[context.commandName!] = commandOutputs[context.commandName!] ?? []; + commandOutputs[context.commandName!].push(r.output); + return r; + }); + + requireRequestsFrom(client) + .toMatch({ + hostname: /amazon/, + }) + .respondWith( + new HttpResponse({ + statusCode: 200, + headers: { + "x-amz-id-2": + "bKbXrk1IXbqfupvsn8gGtkGi33Nszcq9iwiah4xGeydCedkuKWeht6xnBkn0sCBhVDyDs0Xa4ecdbnxtyzMGqc17Cv6Se7P8", + "x-amz-request-id": "MC075MYM6KAT5AQE", + date: "Fri, 26 Sep 2025 17:27:23 GMT", + "x-amz-server-side-encryption": "AES256", + "x-amz-checksum-algorithm": "CRC32", + "x-amz-checksum-type": "COMPOSITE", + "transfer-encoding": "chunked", + server: "AmazonS3", + }, + body: Readable.from( + Buffer.from( + ` + + + sdkreleasev3integtestreso-integtestbucketa93771ae-zh5lrv1xnwjx + + + MPU-1758907641953 + + + FPyQ2V.AnlVZcN3GUqieu5Ael9CllYWycVH0slQyiS9wYjDeUKS0okoMm.jbbbmMbNln.K8HtPzbwjCChgCUH9B94b6MyrD72_auD23tEpXhmel40UtdL.7w_RiNAc1xkyr8ooIKRsGyDE9J.WIH0Q-- + + ` + ) + ), + }), + new HttpResponse({ + statusCode: 200, + headers: { + "x-amz-id-2": + "1ezf9iJF3YvPWo3SF2UCrgFGBXltd25g9bHUA9W4k58PJ/W03OZ13nIOEmGE+NCRCbmuERJ4lvML5zGQU0JCw44sOelC9sRb", + "x-amz-request-id": "MC01ZVGGQ8Z7950C", + date: "Fri, 26 Sep 2025 17:27:23 GMT", + etag: '"eb3760d36bc660b509833238c0799b58"', + "x-amz-checksum-crc32": "Ikyd7A==", + "x-amz-server-side-encryption": "AES256", + "content-length": "0", + server: "AmazonS3", + }, + }), + new HttpResponse({ + statusCode: 200, + headers: { + "x-amz-id-2": "JwgQ5LZ9Sx3fj6G46KOjfx7HI2XvK18Nx6iCOQPH+/UFjbFPju3hlZ7Gq8BIW6g2IiyI8cM3v1LPh+Me8KmCZQ==", + "x-amz-request-id": "MC0DH0R666ESRWE3", + date: "Fri, 26 Sep 2025 17:27:23 GMT", + etag: '"76d0701ab8175448d01476321416bf01"', + "x-amz-checksum-crc32": "JTOG+w==", + "x-amz-server-side-encryption": "AES256", + "content-length": "0", + server: "AmazonS3", + }, + }), + new HttpResponse({ + statusCode: 200, + headers: { + "x-amz-id-2": "0G3bPmuvsW9FMp4OCpbRUxtldh81E3PxbvhUuXsCtasMMpYVfKlxvYkWD9wekOxD/C0xay5ttt/d7MXxz79JBw==", + "x-amz-request-id": "HY3KQKTR6ZKMGZ6E", + date: "Fri, 26 Sep 2025 17:27:24 GMT", + "x-amz-version-id": "PVmXZ_B1Qs3bTot7SY6w_.aiH3TpVbQ6", + "x-amz-server-side-encryption": "AES256", + "content-type": "application/xml", + "transfer-encoding": "chunked", + server: "AmazonS3", + }, + body: Readable.from( + Buffer.from( + ` + + + https://bucket.s3.us-west-2.amazonaws.com/MPU-1758907641953 + + + bucket + + + MPU-1758907641953 + + + "e4cd0558ba33b2b33aa64f158deae527-2" + + + 53GakA==-2 + + + COMPOSITE + + ` + ) + ), + }) + ); + + const uploadOutput = await new Upload({ + client, + params: { + Bucket: "bucket", + Key: "key", + Body: randomBytes(6 * 1024 * 1024), + }, + }).done(); + + /** + * (Because the XML was given without trimming). + */ + function trimObject(item: any): any { + if (typeof item === "string") { + return item.trim(); + } + if (Array.isArray(item)) { + return item.map(trimObject); + } + if (item && typeof item === "object") { + for (const key in item) { + item[key] = trimObject(item[key]); + } + } + return item; + } + trimObject(commandOutputs); + + expect(commandOutputs).toEqual({ + CreateMultipartUploadCommand: [ + { + $metadata: { + httpStatusCode: 200, + requestId: "MC075MYM6KAT5AQE", + extendedRequestId: + "bKbXrk1IXbqfupvsn8gGtkGi33Nszcq9iwiah4xGeydCedkuKWeht6xnBkn0sCBhVDyDs0Xa4ecdbnxtyzMGqc17Cv6Se7P8", + attempts: 1, + totalRetryDelay: 0, + }, + ServerSideEncryption: "AES256", + ChecksumAlgorithm: "CRC32", + ChecksumType: "COMPOSITE", + Bucket: "sdkreleasev3integtestreso-integtestbucketa93771ae-zh5lrv1xnwjx", + Key: "MPU-1758907641953", + UploadId: + "FPyQ2V.AnlVZcN3GUqieu5Ael9CllYWycVH0slQyiS9wYjDeUKS0okoMm.jbbbmMbNln.K8HtPzbwjCChgCUH9B94b6MyrD72_auD23tEpXhmel40UtdL.7w_RiNAc1xkyr8ooIKRsGyDE9J.WIH0Q--", + }, + ], + UploadPartCommand: [ + { + $metadata: { + httpStatusCode: 200, + requestId: "MC01ZVGGQ8Z7950C", + extendedRequestId: + "1ezf9iJF3YvPWo3SF2UCrgFGBXltd25g9bHUA9W4k58PJ/W03OZ13nIOEmGE+NCRCbmuERJ4lvML5zGQU0JCw44sOelC9sRb", + attempts: 1, + totalRetryDelay: 0, + }, + ServerSideEncryption: "AES256", + ETag: '"eb3760d36bc660b509833238c0799b58"', + ChecksumCRC32: "Ikyd7A==", + }, + { + $metadata: { + httpStatusCode: 200, + requestId: "MC0DH0R666ESRWE3", + extendedRequestId: + "JwgQ5LZ9Sx3fj6G46KOjfx7HI2XvK18Nx6iCOQPH+/UFjbFPju3hlZ7Gq8BIW6g2IiyI8cM3v1LPh+Me8KmCZQ==", + attempts: 1, + totalRetryDelay: 0, + }, + ServerSideEncryption: "AES256", + ETag: '"76d0701ab8175448d01476321416bf01"', + ChecksumCRC32: "JTOG+w==", + }, + ], + CompleteMultipartUploadCommand: [ + { + $metadata: { + httpStatusCode: 200, + requestId: "HY3KQKTR6ZKMGZ6E", + extendedRequestId: + "0G3bPmuvsW9FMp4OCpbRUxtldh81E3PxbvhUuXsCtasMMpYVfKlxvYkWD9wekOxD/C0xay5ttt/d7MXxz79JBw==", + attempts: 1, + totalRetryDelay: 0, + }, + ServerSideEncryption: "AES256", + VersionId: "PVmXZ_B1Qs3bTot7SY6w_.aiH3TpVbQ6", + Bucket: "bucket", + ChecksumCRC32: "53GakA==-2", + ChecksumType: "COMPOSITE", + ETag: '"e4cd0558ba33b2b33aa64f158deae527-2"', + Key: "MPU-1758907641953", + Location: "https://bucket.s3.us-west-2.amazonaws.com/MPU-1758907641953", + }, + ], + }); + + expect(uploadOutput).toMatchObject(commandOutputs.CompleteMultipartUploadCommand[0]); + }); +}, 60_000); diff --git a/lib/lib-storage/vitest.config.integ.mts b/lib/lib-storage/vitest.config.integ.mts new file mode 100644 index 0000000000000..5802db1ac64a8 --- /dev/null +++ b/lib/lib-storage/vitest.config.integ.mts @@ -0,0 +1,8 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["**/*.integ.spec.ts"], + environment: "node", + }, +}); From c4e122435c1abac242c637e1f72cff12e75256a4 Mon Sep 17 00:00:00 2001 From: smilkuri Date: Thu, 2 Oct 2025 15:58:41 +0000 Subject: [PATCH 2/4] test(lib-storage): add PutObject response mapping test for Upload --- lib/lib-storage/src/lib-storage.integ.spec.ts | 304 ++++++++++++++++++ 1 file changed, 304 insertions(+) create mode 100644 lib/lib-storage/src/lib-storage.integ.spec.ts diff --git a/lib/lib-storage/src/lib-storage.integ.spec.ts b/lib/lib-storage/src/lib-storage.integ.spec.ts new file mode 100644 index 0000000000000..940d2460c6cad --- /dev/null +++ b/lib/lib-storage/src/lib-storage.integ.spec.ts @@ -0,0 +1,304 @@ +import { requireRequestsFrom } from "@aws-sdk/aws-util-test/src"; +import { S3 } from "@aws-sdk/client-s3"; +import { Upload } from "@aws-sdk/lib-storage"; +import { HttpResponse } from "@smithy/protocol-http"; +import { randomBytes } from "crypto"; +import { Readable } from "node:stream"; +import { describe, expect, test as it } from "vitest"; + +describe("lib storage integration test", () => { + it("verifies CompleteMultipartUpload response is properly mapped to Upload response for large files", async () => { + const client = new S3({ + region: "us-west-2", + credentials: { + accessKeyId: "INTEG", + secretAccessKey: "INTEG", + }, + }); + + const commandOutputs: Record = {}; + + client.middlewareStack.add((next, context) => async (args) => { + const r = await next(args); + commandOutputs[context.commandName!] = commandOutputs[context.commandName!] ?? []; + commandOutputs[context.commandName!].push(r.output); + return r; + }); + + requireRequestsFrom(client) + .toMatch({ + hostname: /amazon/, + }) + .respondWith( + new HttpResponse({ + statusCode: 200, + headers: { + "x-amz-id-2": + "bKbXrk1IXbqfupvsn8gGtkGi33Nszcq9iwiah4xGeydCedkuKWeht6xnBkn0sCBhVDyDs0Xa4ecdbnxtyzMGqc17Cv6Se7P8", + "x-amz-request-id": "MC075MYM6KAT5AQE", + date: "Fri, 26 Sep 2025 17:27:23 GMT", + "x-amz-server-side-encryption": "AES256", + "x-amz-checksum-algorithm": "CRC32", + "x-amz-checksum-type": "COMPOSITE", + "transfer-encoding": "chunked", + server: "AmazonS3", + }, + body: Readable.from( + Buffer.from( + ` + + + sdkreleasev3integtestreso-integtestbucketa93771ae-zh5lrv1xnwjx + + + MPU-1758907641953 + + + FPyQ2V.AnlVZcN3GUqieu5Ael9CllYWycVH0slQyiS9wYjDeUKS0okoMm.jbbbmMbNln.K8HtPzbwjCChgCUH9B94b6MyrD72_auD23tEpXhmel40UtdL.7w_RiNAc1xkyr8ooIKRsGyDE9J.WIH0Q-- + + ` + ) + ), + }), + new HttpResponse({ + statusCode: 200, + headers: { + "x-amz-id-2": + "1ezf9iJF3YvPWo3SF2UCrgFGBXltd25g9bHUA9W4k58PJ/W03OZ13nIOEmGE+NCRCbmuERJ4lvML5zGQU0JCw44sOelC9sRb", + "x-amz-request-id": "MC01ZVGGQ8Z7950C", + date: "Fri, 26 Sep 2025 17:27:23 GMT", + etag: '"eb3760d36bc660b509833238c0799b58"', + "x-amz-checksum-crc32": "Ikyd7A==", + "x-amz-server-side-encryption": "AES256", + "content-length": "0", + server: "AmazonS3", + }, + }), + new HttpResponse({ + statusCode: 200, + headers: { + "x-amz-id-2": "JwgQ5LZ9Sx3fj6G46KOjfx7HI2XvK18Nx6iCOQPH+/UFjbFPju3hlZ7Gq8BIW6g2IiyI8cM3v1LPh+Me8KmCZQ==", + "x-amz-request-id": "MC0DH0R666ESRWE3", + date: "Fri, 26 Sep 2025 17:27:23 GMT", + etag: '"76d0701ab8175448d01476321416bf01"', + "x-amz-checksum-crc32": "JTOG+w==", + "x-amz-server-side-encryption": "AES256", + "content-length": "0", + server: "AmazonS3", + }, + }), + new HttpResponse({ + statusCode: 200, + headers: { + "x-amz-id-2": "0G3bPmuvsW9FMp4OCpbRUxtldh81E3PxbvhUuXsCtasMMpYVfKlxvYkWD9wekOxD/C0xay5ttt/d7MXxz79JBw==", + "x-amz-request-id": "HY3KQKTR6ZKMGZ6E", + date: "Fri, 26 Sep 2025 17:27:24 GMT", + "x-amz-version-id": "PVmXZ_B1Qs3bTot7SY6w_.aiH3TpVbQ6", + "x-amz-server-side-encryption": "AES256", + "content-type": "application/xml", + "transfer-encoding": "chunked", + server: "AmazonS3", + }, + body: Readable.from( + Buffer.from( + ` + + + https://bucket.s3.us-west-2.amazonaws.com/MPU-1758907641953 + + + bucket + + + MPU-1758907641953 + + + "e4cd0558ba33b2b33aa64f158deae527-2" + + + 53GakA==-2 + + + COMPOSITE + + ` + ) + ), + }) + ); + + const uploadOutput = await new Upload({ + client, + params: { + Bucket: "bucket", + Key: "key", + Body: randomBytes(6 * 1024 * 1024), + }, + }).done(); + + /** + * (Because the XML was given without trimming). + */ + function trimObject(item: any): any { + if (typeof item === "string") { + return item.trim(); + } + if (Array.isArray(item)) { + return item.map(trimObject); + } + if (item && typeof item === "object") { + for (const key in item) { + item[key] = trimObject(item[key]); + } + } + return item; + } + trimObject(commandOutputs); + + expect(commandOutputs).toEqual({ + CreateMultipartUploadCommand: [ + { + $metadata: { + httpStatusCode: 200, + requestId: "MC075MYM6KAT5AQE", + extendedRequestId: + "bKbXrk1IXbqfupvsn8gGtkGi33Nszcq9iwiah4xGeydCedkuKWeht6xnBkn0sCBhVDyDs0Xa4ecdbnxtyzMGqc17Cv6Se7P8", + attempts: 1, + totalRetryDelay: 0, + }, + ServerSideEncryption: "AES256", + ChecksumAlgorithm: "CRC32", + ChecksumType: "COMPOSITE", + Bucket: "sdkreleasev3integtestreso-integtestbucketa93771ae-zh5lrv1xnwjx", + Key: "MPU-1758907641953", + UploadId: + "FPyQ2V.AnlVZcN3GUqieu5Ael9CllYWycVH0slQyiS9wYjDeUKS0okoMm.jbbbmMbNln.K8HtPzbwjCChgCUH9B94b6MyrD72_auD23tEpXhmel40UtdL.7w_RiNAc1xkyr8ooIKRsGyDE9J.WIH0Q--", + }, + ], + UploadPartCommand: [ + { + $metadata: { + httpStatusCode: 200, + requestId: "MC01ZVGGQ8Z7950C", + extendedRequestId: + "1ezf9iJF3YvPWo3SF2UCrgFGBXltd25g9bHUA9W4k58PJ/W03OZ13nIOEmGE+NCRCbmuERJ4lvML5zGQU0JCw44sOelC9sRb", + attempts: 1, + totalRetryDelay: 0, + }, + ServerSideEncryption: "AES256", + ETag: '"eb3760d36bc660b509833238c0799b58"', + ChecksumCRC32: "Ikyd7A==", + }, + { + $metadata: { + httpStatusCode: 200, + requestId: "MC0DH0R666ESRWE3", + extendedRequestId: + "JwgQ5LZ9Sx3fj6G46KOjfx7HI2XvK18Nx6iCOQPH+/UFjbFPju3hlZ7Gq8BIW6g2IiyI8cM3v1LPh+Me8KmCZQ==", + attempts: 1, + totalRetryDelay: 0, + }, + ServerSideEncryption: "AES256", + ETag: '"76d0701ab8175448d01476321416bf01"', + ChecksumCRC32: "JTOG+w==", + }, + ], + CompleteMultipartUploadCommand: [ + { + $metadata: { + httpStatusCode: 200, + requestId: "HY3KQKTR6ZKMGZ6E", + extendedRequestId: + "0G3bPmuvsW9FMp4OCpbRUxtldh81E3PxbvhUuXsCtasMMpYVfKlxvYkWD9wekOxD/C0xay5ttt/d7MXxz79JBw==", + attempts: 1, + totalRetryDelay: 0, + }, + ServerSideEncryption: "AES256", + VersionId: "PVmXZ_B1Qs3bTot7SY6w_.aiH3TpVbQ6", + Bucket: "bucket", + ChecksumCRC32: "53GakA==-2", + ChecksumType: "COMPOSITE", + ETag: '"e4cd0558ba33b2b33aa64f158deae527-2"', + Key: "MPU-1758907641953", + Location: "https://bucket.s3.us-west-2.amazonaws.com/MPU-1758907641953", + }, + ], + }); + + expect(uploadOutput).toMatchObject(commandOutputs.CompleteMultipartUploadCommand[0]); + }); + + it("verifies PutObject response is properly mapped to Upload response for small files", async () => { + const client = new S3({ + region: "us-west-2", + credentials: { + accessKeyId: "INTEG", + secretAccessKey: "INTEG", + }, + }); + + const commandOutputs: Record = {}; + + client.middlewareStack.add((next, context) => async (args) => { + const r = await next(args); + commandOutputs[context.commandName!] = commandOutputs[context.commandName!] ?? []; + commandOutputs[context.commandName!].push(r.output); + return r; + }); + + requireRequestsFrom(client) + .toMatch({ + hostname: /amazon/, + }) + .respondWith( + new HttpResponse({ + statusCode: 200, + headers: { + "x-amz-id-2": + "abc123def456ghi789jkl012mno345pqr678stu901vwx234yzA567BCD890EFG123HIJ456KLM789NOP012QRS345TUV", + "x-amz-request-id": "ABCD1234EFGH5678", + date: "Fri, 26 Sep 2025 17:30:00 GMT", + etag: '"d41d8cd98f00b204e9800998ecf8427e"', + "x-amz-checksum-crc32": "AAAAAA==", + "x-amz-checksum-type": "CRC32", + "x-amz-server-side-encryption": "AES256", + "x-amz-version-id": "null", + "content-length": "0", + server: "AmazonS3", + }, + }) + ); + + // Use small data to trigger PutObject instead of multipart upload + const uploadOutput = await new Upload({ + client, + params: { + Bucket: "bucket", + Key: "small-file-key", + Body: randomBytes(1024), // 1KB - small enough to use PutObject + }, + }).done(); + + expect(commandOutputs).toEqual({ + PutObjectCommand: [ + { + $metadata: { + httpStatusCode: 200, + requestId: "ABCD1234EFGH5678", + extendedRequestId: + "abc123def456ghi789jkl012mno345pqr678stu901vwx234yzA567BCD890EFG123HIJ456KLM789NOP012QRS345TUV", + attempts: 1, + totalRetryDelay: 0, + }, + ETag: '"d41d8cd98f00b204e9800998ecf8427e"', + ChecksumCRC32: "AAAAAA==", + ChecksumType: "CRC32", + ServerSideEncryption: "AES256", + VersionId: "null", + }, + ], + }); + + expect(uploadOutput).toMatchObject(commandOutputs.PutObjectCommand[0]); + }); +}, 60_000); From 43a7d6d03e91b410b3014310001c807caa9f5f01 Mon Sep 17 00:00:00 2001 From: smilkuri Date: Thu, 2 Oct 2025 16:25:22 +0000 Subject: [PATCH 3/4] test(lib-storage): remove duplicate file --- .../src/lib-storage-example.integ.spec.ts | 125 ++++--- lib/lib-storage/src/lib-storage.integ.spec.ts | 304 ------------------ 2 files changed, 74 insertions(+), 355 deletions(-) delete mode 100644 lib/lib-storage/src/lib-storage.integ.spec.ts diff --git a/lib/lib-storage/src/lib-storage-example.integ.spec.ts b/lib/lib-storage/src/lib-storage-example.integ.spec.ts index f27ad6bdc938c..2b7176752acc3 100644 --- a/lib/lib-storage/src/lib-storage-example.integ.spec.ts +++ b/lib/lib-storage/src/lib-storage-example.integ.spec.ts @@ -1,62 +1,12 @@ -import { getE2eTestResources, requireRequestsFrom } from "@aws-sdk/aws-util-test/src"; +import { requireRequestsFrom } from "@aws-sdk/aws-util-test/src"; import { S3 } from "@aws-sdk/client-s3"; import { Upload } from "@aws-sdk/lib-storage"; -import { getHttpDebugLogPlugin } from "@aws-sdk/middleware-http-debug-log/src"; import { HttpResponse } from "@smithy/protocol-http"; import { randomBytes } from "crypto"; import { Readable } from "node:stream"; import { describe, expect, test as it } from "vitest"; describe("lib storage integration test", () => { - const example = async () => { - const e2eTestResourcesEnv = await getE2eTestResources(); - Object.assign(process.env, e2eTestResourcesEnv); - - const region = process?.env?.AWS_SMOKE_TEST_REGION as string; - const Bucket = process?.env?.AWS_SMOKE_TEST_BUCKET as string; - const data = randomBytes(6 * 1024 * 1024); - - const s3 = new S3({ - region, - }); - const Key = `MPU-${Date.now()}`; - - const client = new S3({ - region, - }); - // This will print out all requests and responses so you can use - // them in an integration mock later. - client.middlewareStack.use( - getHttpDebugLogPlugin({ - request: { - url: true, - command: true, - method: true, - }, - response: { - statusCode: true, - headers: true, - body: true, - formatBody: true, - }, - }) - ); - - await new Upload({ - client, - params: { - Bucket, - Key, - Body: data, - }, - }).done(); - - await s3.deleteObject({ - Bucket, - Key, - }); - }; - it("example of how to write an integration test that includes responses", async () => { const client = new S3({ region: "us-west-2", @@ -277,4 +227,77 @@ describe("lib storage integration test", () => { expect(uploadOutput).toMatchObject(commandOutputs.CompleteMultipartUploadCommand[0]); }); + + it("verifies PutObject response is properly mapped to Upload response for small files", async () => { + const client = new S3({ + region: "us-west-2", + credentials: { + accessKeyId: "INTEG", + secretAccessKey: "INTEG", + }, + }); + + const commandOutputs: Record = {}; + + client.middlewareStack.add((next, context) => async (args) => { + const r = await next(args); + commandOutputs[context.commandName!] = commandOutputs[context.commandName!] ?? []; + commandOutputs[context.commandName!].push(r.output); + return r; + }); + + requireRequestsFrom(client) + .toMatch({ + hostname: /amazon/, + }) + .respondWith( + new HttpResponse({ + statusCode: 200, + headers: { + "x-amz-id-2": + "abc123def456ghi789jkl012mno345pqr678stu901vwx234yzA567BCD890EFG123HIJ456KLM789NOP012QRS345TUV", + "x-amz-request-id": "ABCD1234EFGH5678", + date: "Fri, 26 Sep 2025 17:30:00 GMT", + etag: '"d41d8cd98f00b204e9800998ecf8427e"', + "x-amz-checksum-crc32": "AAAAAA==", + "x-amz-checksum-type": "CRC32", + "x-amz-server-side-encryption": "AES256", + "x-amz-version-id": "null", + "content-length": "0", + server: "AmazonS3", + }, + }) + ); + + const uploadOutput = await new Upload({ + client, + params: { + Bucket: "bucket", + Key: "small-file-key", + Body: randomBytes(1024), // 1KB - small enough to use PutObject + }, + }).done(); + + expect(commandOutputs).toEqual({ + PutObjectCommand: [ + { + $metadata: { + httpStatusCode: 200, + requestId: "ABCD1234EFGH5678", + extendedRequestId: + "abc123def456ghi789jkl012mno345pqr678stu901vwx234yzA567BCD890EFG123HIJ456KLM789NOP012QRS345TUV", + attempts: 1, + totalRetryDelay: 0, + }, + ETag: '"d41d8cd98f00b204e9800998ecf8427e"', + ChecksumCRC32: "AAAAAA==", + ChecksumType: "CRC32", + ServerSideEncryption: "AES256", + VersionId: "null", + }, + ], + }); + + expect(uploadOutput).toMatchObject(commandOutputs.PutObjectCommand[0]); + }); }, 60_000); diff --git a/lib/lib-storage/src/lib-storage.integ.spec.ts b/lib/lib-storage/src/lib-storage.integ.spec.ts deleted file mode 100644 index 940d2460c6cad..0000000000000 --- a/lib/lib-storage/src/lib-storage.integ.spec.ts +++ /dev/null @@ -1,304 +0,0 @@ -import { requireRequestsFrom } from "@aws-sdk/aws-util-test/src"; -import { S3 } from "@aws-sdk/client-s3"; -import { Upload } from "@aws-sdk/lib-storage"; -import { HttpResponse } from "@smithy/protocol-http"; -import { randomBytes } from "crypto"; -import { Readable } from "node:stream"; -import { describe, expect, test as it } from "vitest"; - -describe("lib storage integration test", () => { - it("verifies CompleteMultipartUpload response is properly mapped to Upload response for large files", async () => { - const client = new S3({ - region: "us-west-2", - credentials: { - accessKeyId: "INTEG", - secretAccessKey: "INTEG", - }, - }); - - const commandOutputs: Record = {}; - - client.middlewareStack.add((next, context) => async (args) => { - const r = await next(args); - commandOutputs[context.commandName!] = commandOutputs[context.commandName!] ?? []; - commandOutputs[context.commandName!].push(r.output); - return r; - }); - - requireRequestsFrom(client) - .toMatch({ - hostname: /amazon/, - }) - .respondWith( - new HttpResponse({ - statusCode: 200, - headers: { - "x-amz-id-2": - "bKbXrk1IXbqfupvsn8gGtkGi33Nszcq9iwiah4xGeydCedkuKWeht6xnBkn0sCBhVDyDs0Xa4ecdbnxtyzMGqc17Cv6Se7P8", - "x-amz-request-id": "MC075MYM6KAT5AQE", - date: "Fri, 26 Sep 2025 17:27:23 GMT", - "x-amz-server-side-encryption": "AES256", - "x-amz-checksum-algorithm": "CRC32", - "x-amz-checksum-type": "COMPOSITE", - "transfer-encoding": "chunked", - server: "AmazonS3", - }, - body: Readable.from( - Buffer.from( - ` - - - sdkreleasev3integtestreso-integtestbucketa93771ae-zh5lrv1xnwjx - - - MPU-1758907641953 - - - FPyQ2V.AnlVZcN3GUqieu5Ael9CllYWycVH0slQyiS9wYjDeUKS0okoMm.jbbbmMbNln.K8HtPzbwjCChgCUH9B94b6MyrD72_auD23tEpXhmel40UtdL.7w_RiNAc1xkyr8ooIKRsGyDE9J.WIH0Q-- - - ` - ) - ), - }), - new HttpResponse({ - statusCode: 200, - headers: { - "x-amz-id-2": - "1ezf9iJF3YvPWo3SF2UCrgFGBXltd25g9bHUA9W4k58PJ/W03OZ13nIOEmGE+NCRCbmuERJ4lvML5zGQU0JCw44sOelC9sRb", - "x-amz-request-id": "MC01ZVGGQ8Z7950C", - date: "Fri, 26 Sep 2025 17:27:23 GMT", - etag: '"eb3760d36bc660b509833238c0799b58"', - "x-amz-checksum-crc32": "Ikyd7A==", - "x-amz-server-side-encryption": "AES256", - "content-length": "0", - server: "AmazonS3", - }, - }), - new HttpResponse({ - statusCode: 200, - headers: { - "x-amz-id-2": "JwgQ5LZ9Sx3fj6G46KOjfx7HI2XvK18Nx6iCOQPH+/UFjbFPju3hlZ7Gq8BIW6g2IiyI8cM3v1LPh+Me8KmCZQ==", - "x-amz-request-id": "MC0DH0R666ESRWE3", - date: "Fri, 26 Sep 2025 17:27:23 GMT", - etag: '"76d0701ab8175448d01476321416bf01"', - "x-amz-checksum-crc32": "JTOG+w==", - "x-amz-server-side-encryption": "AES256", - "content-length": "0", - server: "AmazonS3", - }, - }), - new HttpResponse({ - statusCode: 200, - headers: { - "x-amz-id-2": "0G3bPmuvsW9FMp4OCpbRUxtldh81E3PxbvhUuXsCtasMMpYVfKlxvYkWD9wekOxD/C0xay5ttt/d7MXxz79JBw==", - "x-amz-request-id": "HY3KQKTR6ZKMGZ6E", - date: "Fri, 26 Sep 2025 17:27:24 GMT", - "x-amz-version-id": "PVmXZ_B1Qs3bTot7SY6w_.aiH3TpVbQ6", - "x-amz-server-side-encryption": "AES256", - "content-type": "application/xml", - "transfer-encoding": "chunked", - server: "AmazonS3", - }, - body: Readable.from( - Buffer.from( - ` - - - https://bucket.s3.us-west-2.amazonaws.com/MPU-1758907641953 - - - bucket - - - MPU-1758907641953 - - - "e4cd0558ba33b2b33aa64f158deae527-2" - - - 53GakA==-2 - - - COMPOSITE - - ` - ) - ), - }) - ); - - const uploadOutput = await new Upload({ - client, - params: { - Bucket: "bucket", - Key: "key", - Body: randomBytes(6 * 1024 * 1024), - }, - }).done(); - - /** - * (Because the XML was given without trimming). - */ - function trimObject(item: any): any { - if (typeof item === "string") { - return item.trim(); - } - if (Array.isArray(item)) { - return item.map(trimObject); - } - if (item && typeof item === "object") { - for (const key in item) { - item[key] = trimObject(item[key]); - } - } - return item; - } - trimObject(commandOutputs); - - expect(commandOutputs).toEqual({ - CreateMultipartUploadCommand: [ - { - $metadata: { - httpStatusCode: 200, - requestId: "MC075MYM6KAT5AQE", - extendedRequestId: - "bKbXrk1IXbqfupvsn8gGtkGi33Nszcq9iwiah4xGeydCedkuKWeht6xnBkn0sCBhVDyDs0Xa4ecdbnxtyzMGqc17Cv6Se7P8", - attempts: 1, - totalRetryDelay: 0, - }, - ServerSideEncryption: "AES256", - ChecksumAlgorithm: "CRC32", - ChecksumType: "COMPOSITE", - Bucket: "sdkreleasev3integtestreso-integtestbucketa93771ae-zh5lrv1xnwjx", - Key: "MPU-1758907641953", - UploadId: - "FPyQ2V.AnlVZcN3GUqieu5Ael9CllYWycVH0slQyiS9wYjDeUKS0okoMm.jbbbmMbNln.K8HtPzbwjCChgCUH9B94b6MyrD72_auD23tEpXhmel40UtdL.7w_RiNAc1xkyr8ooIKRsGyDE9J.WIH0Q--", - }, - ], - UploadPartCommand: [ - { - $metadata: { - httpStatusCode: 200, - requestId: "MC01ZVGGQ8Z7950C", - extendedRequestId: - "1ezf9iJF3YvPWo3SF2UCrgFGBXltd25g9bHUA9W4k58PJ/W03OZ13nIOEmGE+NCRCbmuERJ4lvML5zGQU0JCw44sOelC9sRb", - attempts: 1, - totalRetryDelay: 0, - }, - ServerSideEncryption: "AES256", - ETag: '"eb3760d36bc660b509833238c0799b58"', - ChecksumCRC32: "Ikyd7A==", - }, - { - $metadata: { - httpStatusCode: 200, - requestId: "MC0DH0R666ESRWE3", - extendedRequestId: - "JwgQ5LZ9Sx3fj6G46KOjfx7HI2XvK18Nx6iCOQPH+/UFjbFPju3hlZ7Gq8BIW6g2IiyI8cM3v1LPh+Me8KmCZQ==", - attempts: 1, - totalRetryDelay: 0, - }, - ServerSideEncryption: "AES256", - ETag: '"76d0701ab8175448d01476321416bf01"', - ChecksumCRC32: "JTOG+w==", - }, - ], - CompleteMultipartUploadCommand: [ - { - $metadata: { - httpStatusCode: 200, - requestId: "HY3KQKTR6ZKMGZ6E", - extendedRequestId: - "0G3bPmuvsW9FMp4OCpbRUxtldh81E3PxbvhUuXsCtasMMpYVfKlxvYkWD9wekOxD/C0xay5ttt/d7MXxz79JBw==", - attempts: 1, - totalRetryDelay: 0, - }, - ServerSideEncryption: "AES256", - VersionId: "PVmXZ_B1Qs3bTot7SY6w_.aiH3TpVbQ6", - Bucket: "bucket", - ChecksumCRC32: "53GakA==-2", - ChecksumType: "COMPOSITE", - ETag: '"e4cd0558ba33b2b33aa64f158deae527-2"', - Key: "MPU-1758907641953", - Location: "https://bucket.s3.us-west-2.amazonaws.com/MPU-1758907641953", - }, - ], - }); - - expect(uploadOutput).toMatchObject(commandOutputs.CompleteMultipartUploadCommand[0]); - }); - - it("verifies PutObject response is properly mapped to Upload response for small files", async () => { - const client = new S3({ - region: "us-west-2", - credentials: { - accessKeyId: "INTEG", - secretAccessKey: "INTEG", - }, - }); - - const commandOutputs: Record = {}; - - client.middlewareStack.add((next, context) => async (args) => { - const r = await next(args); - commandOutputs[context.commandName!] = commandOutputs[context.commandName!] ?? []; - commandOutputs[context.commandName!].push(r.output); - return r; - }); - - requireRequestsFrom(client) - .toMatch({ - hostname: /amazon/, - }) - .respondWith( - new HttpResponse({ - statusCode: 200, - headers: { - "x-amz-id-2": - "abc123def456ghi789jkl012mno345pqr678stu901vwx234yzA567BCD890EFG123HIJ456KLM789NOP012QRS345TUV", - "x-amz-request-id": "ABCD1234EFGH5678", - date: "Fri, 26 Sep 2025 17:30:00 GMT", - etag: '"d41d8cd98f00b204e9800998ecf8427e"', - "x-amz-checksum-crc32": "AAAAAA==", - "x-amz-checksum-type": "CRC32", - "x-amz-server-side-encryption": "AES256", - "x-amz-version-id": "null", - "content-length": "0", - server: "AmazonS3", - }, - }) - ); - - // Use small data to trigger PutObject instead of multipart upload - const uploadOutput = await new Upload({ - client, - params: { - Bucket: "bucket", - Key: "small-file-key", - Body: randomBytes(1024), // 1KB - small enough to use PutObject - }, - }).done(); - - expect(commandOutputs).toEqual({ - PutObjectCommand: [ - { - $metadata: { - httpStatusCode: 200, - requestId: "ABCD1234EFGH5678", - extendedRequestId: - "abc123def456ghi789jkl012mno345pqr678stu901vwx234yzA567BCD890EFG123HIJ456KLM789NOP012QRS345TUV", - attempts: 1, - totalRetryDelay: 0, - }, - ETag: '"d41d8cd98f00b204e9800998ecf8427e"', - ChecksumCRC32: "AAAAAA==", - ChecksumType: "CRC32", - ServerSideEncryption: "AES256", - VersionId: "null", - }, - ], - }); - - expect(uploadOutput).toMatchObject(commandOutputs.PutObjectCommand[0]); - }); -}, 60_000); From 06ed41f1fcbc4857e978689781c60ccd8ab0e92c Mon Sep 17 00:00:00 2001 From: smilkuri Date: Thu, 2 Oct 2025 16:30:59 +0000 Subject: [PATCH 4/4] test(lib-storage): rename the test file --- ...-storage-example.integ.spec.ts => lib-storage.integ.spec.ts} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename lib/lib-storage/src/{lib-storage-example.integ.spec.ts => lib-storage.integ.spec.ts} (98%) diff --git a/lib/lib-storage/src/lib-storage-example.integ.spec.ts b/lib/lib-storage/src/lib-storage.integ.spec.ts similarity index 98% rename from lib/lib-storage/src/lib-storage-example.integ.spec.ts rename to lib/lib-storage/src/lib-storage.integ.spec.ts index 2b7176752acc3..dde522591fa0d 100644 --- a/lib/lib-storage/src/lib-storage-example.integ.spec.ts +++ b/lib/lib-storage/src/lib-storage.integ.spec.ts @@ -7,7 +7,7 @@ import { Readable } from "node:stream"; import { describe, expect, test as it } from "vitest"; describe("lib storage integration test", () => { - it("example of how to write an integration test that includes responses", async () => { + it("verifies CompleteMultipartUpload response is properly mapped to Upload response for large files", async () => { const client = new S3({ region: "us-west-2", credentials: {