From fabddf0c8ea557cad160f9ac41238cf6998357c4 Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Fri, 23 Dec 2022 12:04:36 +0100 Subject: [PATCH 01/34] WIP docker bundle with cp just a POC so far --- packages/@aws-cdk/core/lib/asset-staging.ts | 43 ++++++++++++++------- packages/@aws-cdk/core/lib/bundling.ts | 3 +- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/packages/@aws-cdk/core/lib/asset-staging.ts b/packages/@aws-cdk/core/lib/asset-staging.ts index 6831790d50938..e066aee27cdf7 100644 --- a/packages/@aws-cdk/core/lib/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/asset-staging.ts @@ -5,7 +5,7 @@ import * as cxapi from '@aws-cdk/cx-api'; import { Construct } from 'constructs'; import * as fs from 'fs-extra'; import { AssetHashType, AssetOptions, FileAssetPackaging } from './assets'; -import { BundlingOptions, BundlingOutput } from './bundling'; +import { BundlingOptions, BundlingOutput, dockerExec } from './bundling'; import { FileSystem, FingerprintOptions } from './fs'; import { clearLargeFileFingerprintCache } from './fs/fingerprint'; import { Names } from './names'; @@ -433,17 +433,17 @@ export class AssetStaging extends Construct { fs.chmodSync(bundleDir, 0o777); // Always mount input and output dir - const volumes = [ - { - hostPath: this.sourcePath, - containerPath: AssetStaging.BUNDLING_INPUT_DIR, - }, - { - hostPath: bundleDir, - containerPath: AssetStaging.BUNDLING_OUTPUT_DIR, - }, - ...options.volumes ?? [], - ]; + // const volumes = [ + // { + // hostPath: this.sourcePath, + // containerPath: AssetStaging.BUNDLING_INPUT_DIR, + // }, + // { + // hostPath: bundleDir, + // containerPath: AssetStaging.BUNDLING_OUTPUT_DIR, + // }, + // ...options.volumes ?? [], + // ]; let localBundling: boolean | undefined; try { @@ -461,16 +461,31 @@ export class AssetStaging extends Construct { : '1000:1000'; } + const copySuffix = crypto.randomBytes(12).toString('hex'); + const inputVolumeName = `assetInput${copySuffix}`; + const outputVolumeName = `assetOutput${copySuffix}`; + const copyContainerName = `copy${copySuffix}`; + dockerExec(['volume', 'create', inputVolumeName]); + dockerExec(['volume', 'create', outputVolumeName]); + + dockerExec(['run', '--name', copyContainerName, '-v', `${inputVolumeName}:${AssetStaging.BUNDLING_INPUT_DIR}`, '-v', `${outputVolumeName}:${AssetStaging.BUNDLING_OUTPUT_DIR}`, 'alpine', 'sh', '-c', `mkdir -p ${AssetStaging.BUNDLING_INPUT_DIR} && chown -R ${user} ${AssetStaging.BUNDLING_OUTPUT_DIR} && chown -R ${user} ${AssetStaging.BUNDLING_INPUT_DIR}`]); + dockerExec(['cp', `${this.sourcePath}/`, `${copyContainerName}:${AssetStaging.BUNDLING_INPUT_DIR}/`]); + // this always copies to a subfolder, wildcard does not work. not sure how to solve, as name of folder is not known + + options.image.run({ command: options.command, user, - volumes, environment: options.environment, entrypoint: options.entrypoint, workingDirectory: options.workingDirectory ?? AssetStaging.BUNDLING_INPUT_DIR, securityOpt: options.securityOpt ?? '', - volumesFrom: options.volumesFrom, + volumesFrom: [copyContainerName, ...options.volumesFrom ?? []], }); + dockerExec(['cp', `${copyContainerName}:${AssetStaging.BUNDLING_OUTPUT_DIR}`, bundleDir]); + dockerExec(['rm', copyContainerName]); + dockerExec(['volume', 'rm', inputVolumeName]); + dockerExec(['volume', 'rm', outputVolumeName]); } } catch (err) { // When bundling fails, keep the bundle output for diagnosability, but diff --git a/packages/@aws-cdk/core/lib/bundling.ts b/packages/@aws-cdk/core/lib/bundling.ts index a78f5e8a6aca0..0571ae8e56037 100644 --- a/packages/@aws-cdk/core/lib/bundling.ts +++ b/packages/@aws-cdk/core/lib/bundling.ts @@ -383,6 +383,7 @@ export class DockerImage extends BundlingDockerImage { reset(deprecated); return result; } + } /** @@ -536,7 +537,7 @@ function flatten(x: string[][]) { return Array.prototype.concat([], ...x); } -function dockerExec(args: string[], options?: SpawnSyncOptions) { +export function dockerExec(args: string[], options?: SpawnSyncOptions) { const prog = process.env.CDK_DOCKER ?? 'docker'; const proc = spawnSync(prog, args, options ?? { stdio: [ // show Docker output From 025e8d9cb428d748b0ec39e4a3e8a9c35510bcfa Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Fri, 23 Dec 2022 14:07:42 +0100 Subject: [PATCH 02/34] wip mv --- packages/@aws-cdk/core/lib/asset-staging.ts | 24 +++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/core/lib/asset-staging.ts b/packages/@aws-cdk/core/lib/asset-staging.ts index e066aee27cdf7..52bb141100b1a 100644 --- a/packages/@aws-cdk/core/lib/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/asset-staging.ts @@ -468,9 +468,29 @@ export class AssetStaging extends Construct { dockerExec(['volume', 'create', inputVolumeName]); dockerExec(['volume', 'create', outputVolumeName]); - dockerExec(['run', '--name', copyContainerName, '-v', `${inputVolumeName}:${AssetStaging.BUNDLING_INPUT_DIR}`, '-v', `${outputVolumeName}:${AssetStaging.BUNDLING_OUTPUT_DIR}`, 'alpine', 'sh', '-c', `mkdir -p ${AssetStaging.BUNDLING_INPUT_DIR} && chown -R ${user} ${AssetStaging.BUNDLING_OUTPUT_DIR} && chown -R ${user} ${AssetStaging.BUNDLING_INPUT_DIR}`]); + dockerExec([ + 'run', + '--name', copyContainerName, + '-v', `${inputVolumeName}:${AssetStaging.BUNDLING_INPUT_DIR}`, + '-v', `${outputVolumeName}:${AssetStaging.BUNDLING_OUTPUT_DIR}`, + 'alpine', + 'sh', + '-c', + `mkdir -p ${AssetStaging.BUNDLING_INPUT_DIR} && chown -R ${user} ${AssetStaging.BUNDLING_OUTPUT_DIR} && chown -R ${user} ${AssetStaging.BUNDLING_INPUT_DIR}`, + ]); + // copy files over dockerExec(['cp', `${this.sourcePath}/`, `${copyContainerName}:${AssetStaging.BUNDLING_INPUT_DIR}/`]); - // this always copies to a subfolder, wildcard does not work. not sure how to solve, as name of folder is not known + // but docker cp does not behave like normal cp, so we need to move them again + dockerExec([ + 'run', + '--rm', + '-v', `${inputVolumeName}:${AssetStaging.BUNDLING_INPUT_DIR}`, + 'ubuntu', + 'bash', + '-c', + `tmpfolder=$(ls -d ${AssetStaging.BUNDLING_INPUT_DIR}/\*) mv $tmpfolder/\* ${AssetStaging.BUNDLING_INPUT_DIR}/ && rmdir $tmpfolder`, + ]); + // this seems not work always options.image.run({ From c2e10b8861cf279b45a829e7269dfd23cf2a0d25 Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Mon, 2 Jan 2023 12:48:44 +0100 Subject: [PATCH 03/34] adjust cp command --- packages/@aws-cdk/core/lib/asset-staging.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/@aws-cdk/core/lib/asset-staging.ts b/packages/@aws-cdk/core/lib/asset-staging.ts index 52bb141100b1a..566f07f65a521 100644 --- a/packages/@aws-cdk/core/lib/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/asset-staging.ts @@ -479,17 +479,17 @@ export class AssetStaging extends Construct { `mkdir -p ${AssetStaging.BUNDLING_INPUT_DIR} && chown -R ${user} ${AssetStaging.BUNDLING_OUTPUT_DIR} && chown -R ${user} ${AssetStaging.BUNDLING_INPUT_DIR}`, ]); // copy files over - dockerExec(['cp', `${this.sourcePath}/`, `${copyContainerName}:${AssetStaging.BUNDLING_INPUT_DIR}/`]); + dockerExec(['cp', `${this.sourcePath}/.`, `${copyContainerName}:${AssetStaging.BUNDLING_INPUT_DIR}`]); // but docker cp does not behave like normal cp, so we need to move them again - dockerExec([ - 'run', - '--rm', - '-v', `${inputVolumeName}:${AssetStaging.BUNDLING_INPUT_DIR}`, - 'ubuntu', - 'bash', - '-c', - `tmpfolder=$(ls -d ${AssetStaging.BUNDLING_INPUT_DIR}/\*) mv $tmpfolder/\* ${AssetStaging.BUNDLING_INPUT_DIR}/ && rmdir $tmpfolder`, - ]); + // dockerExec([ + // 'run', + // '--rm', + // '-v', `${inputVolumeName}:${AssetStaging.BUNDLING_INPUT_DIR}`, + // 'ubuntu', + // 'bash', + // '-c', + // `tmpfolder=$(ls -d ${AssetStaging.BUNDLING_INPUT_DIR}/\*) mv $tmpfolder/\* ${AssetStaging.BUNDLING_INPUT_DIR}/ && rmdir $tmpfolder`, + // ]); // this seems not work always From 8cb6259c319ae8a114af096796810336b37b3e37 Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Tue, 3 Jan 2023 14:59:30 +0100 Subject: [PATCH 04/34] fix missing path to copy out --- packages/@aws-cdk/core/lib/asset-staging.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/core/lib/asset-staging.ts b/packages/@aws-cdk/core/lib/asset-staging.ts index 566f07f65a521..7c797f7cd9337 100644 --- a/packages/@aws-cdk/core/lib/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/asset-staging.ts @@ -502,7 +502,7 @@ export class AssetStaging extends Construct { securityOpt: options.securityOpt ?? '', volumesFrom: [copyContainerName, ...options.volumesFrom ?? []], }); - dockerExec(['cp', `${copyContainerName}:${AssetStaging.BUNDLING_OUTPUT_DIR}`, bundleDir]); + dockerExec(['cp', `${copyContainerName}:${AssetStaging.BUNDLING_OUTPUT_DIR}/.`, bundleDir]); dockerExec(['rm', copyContainerName]); dockerExec(['volume', 'rm', inputVolumeName]); dockerExec(['volume', 'rm', outputVolumeName]); From c006750c80ed7175f09e921ab7951193bac989e0 Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Wed, 4 Jan 2023 08:19:17 +0100 Subject: [PATCH 05/34] slight cleanup and first failed tests --- packages/@aws-cdk/core/lib/asset-staging.ts | 88 +++++++++++---------- packages/@aws-cdk/core/lib/bundling.ts | 25 ++++++ packages/@aws-cdk/core/test/staging.test.ts | 34 ++++++++ 3 files changed, 104 insertions(+), 43 deletions(-) diff --git a/packages/@aws-cdk/core/lib/asset-staging.ts b/packages/@aws-cdk/core/lib/asset-staging.ts index 7c797f7cd9337..e39facfb36f25 100644 --- a/packages/@aws-cdk/core/lib/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/asset-staging.ts @@ -5,7 +5,7 @@ import * as cxapi from '@aws-cdk/cx-api'; import { Construct } from 'constructs'; import * as fs from 'fs-extra'; import { AssetHashType, AssetOptions, FileAssetPackaging } from './assets'; -import { BundlingOptions, BundlingOutput, dockerExec } from './bundling'; +import { BundlingFileCopyVariant, BundlingOptions, BundlingOutput, dockerExec, DockerVolume } from './bundling'; import { FileSystem, FingerprintOptions } from './fs'; import { clearLargeFileFingerprintCache } from './fs/fingerprint'; import { Names } from './names'; @@ -433,17 +433,22 @@ export class AssetStaging extends Construct { fs.chmodSync(bundleDir, 0o777); // Always mount input and output dir - // const volumes = [ - // { - // hostPath: this.sourcePath, - // containerPath: AssetStaging.BUNDLING_INPUT_DIR, - // }, - // { - // hostPath: bundleDir, - // containerPath: AssetStaging.BUNDLING_OUTPUT_DIR, - // }, - // ...options.volumes ?? [], - // ]; + let volumes: DockerVolume[] = []; + if (options.fileCopyVariant?.valueOf() === BundlingFileCopyVariant.DOCKER_COPY.valueOf()) { + volumes = options.volumes ?? []; + } else { + volumes = [ + { + hostPath: this.sourcePath, + containerPath: AssetStaging.BUNDLING_INPUT_DIR, + }, + { + hostPath: bundleDir, + containerPath: AssetStaging.BUNDLING_OUTPUT_DIR, + }, + ...options.volumes ?? [], + ]; + } let localBundling: boolean | undefined; try { @@ -465,33 +470,26 @@ export class AssetStaging extends Construct { const inputVolumeName = `assetInput${copySuffix}`; const outputVolumeName = `assetOutput${copySuffix}`; const copyContainerName = `copy${copySuffix}`; - dockerExec(['volume', 'create', inputVolumeName]); - dockerExec(['volume', 'create', outputVolumeName]); - - dockerExec([ - 'run', - '--name', copyContainerName, - '-v', `${inputVolumeName}:${AssetStaging.BUNDLING_INPUT_DIR}`, - '-v', `${outputVolumeName}:${AssetStaging.BUNDLING_OUTPUT_DIR}`, - 'alpine', - 'sh', - '-c', - `mkdir -p ${AssetStaging.BUNDLING_INPUT_DIR} && chown -R ${user} ${AssetStaging.BUNDLING_OUTPUT_DIR} && chown -R ${user} ${AssetStaging.BUNDLING_INPUT_DIR}`, - ]); - // copy files over - dockerExec(['cp', `${this.sourcePath}/.`, `${copyContainerName}:${AssetStaging.BUNDLING_INPUT_DIR}`]); - // but docker cp does not behave like normal cp, so we need to move them again - // dockerExec([ - // 'run', - // '--rm', - // '-v', `${inputVolumeName}:${AssetStaging.BUNDLING_INPUT_DIR}`, - // 'ubuntu', - // 'bash', - // '-c', - // `tmpfolder=$(ls -d ${AssetStaging.BUNDLING_INPUT_DIR}/\*) mv $tmpfolder/\* ${AssetStaging.BUNDLING_INPUT_DIR}/ && rmdir $tmpfolder`, - // ]); - // this seems not work always + let volumesFrom: string[] = options.volumesFrom ?? []; + + if (options.fileCopyVariant?.valueOf() === BundlingFileCopyVariant.DOCKER_COPY.valueOf()) { + volumesFrom = [copyContainerName, ...options.volumesFrom ?? []]; + dockerExec(['volume', 'create', inputVolumeName]); + dockerExec(['volume', 'create', outputVolumeName]); + + dockerExec([ + 'run', + '--name', copyContainerName, + '-v', `${inputVolumeName}:${AssetStaging.BUNDLING_INPUT_DIR}`, + '-v', `${outputVolumeName}:${AssetStaging.BUNDLING_OUTPUT_DIR}`, + 'alpine', + 'sh', + '-c', + `mkdir -p ${AssetStaging.BUNDLING_INPUT_DIR} && chown -R ${user} ${AssetStaging.BUNDLING_OUTPUT_DIR} && chown -R ${user} ${AssetStaging.BUNDLING_INPUT_DIR}`, + ]); + dockerExec(['cp', `${this.sourcePath}/.`, `${copyContainerName}:${AssetStaging.BUNDLING_INPUT_DIR}`]); + } options.image.run({ command: options.command, @@ -500,12 +498,16 @@ export class AssetStaging extends Construct { entrypoint: options.entrypoint, workingDirectory: options.workingDirectory ?? AssetStaging.BUNDLING_INPUT_DIR, securityOpt: options.securityOpt ?? '', - volumesFrom: [copyContainerName, ...options.volumesFrom ?? []], + volumes, + volumesFrom, }); - dockerExec(['cp', `${copyContainerName}:${AssetStaging.BUNDLING_OUTPUT_DIR}/.`, bundleDir]); - dockerExec(['rm', copyContainerName]); - dockerExec(['volume', 'rm', inputVolumeName]); - dockerExec(['volume', 'rm', outputVolumeName]); + + if (options.fileCopyVariant?.valueOf() === BundlingFileCopyVariant.DOCKER_COPY.valueOf()) { + dockerExec(['cp', `${copyContainerName}:${AssetStaging.BUNDLING_OUTPUT_DIR}/.`, bundleDir]); + dockerExec(['rm', copyContainerName]); + dockerExec(['volume', 'rm', inputVolumeName]); + dockerExec(['volume', 'rm', outputVolumeName]); + } } } catch (err) { // When bundling fails, keep the bundle output for diagnosability, but diff --git a/packages/@aws-cdk/core/lib/bundling.ts b/packages/@aws-cdk/core/lib/bundling.ts index 0571ae8e56037..553a8834e1b05 100644 --- a/packages/@aws-cdk/core/lib/bundling.ts +++ b/packages/@aws-cdk/core/lib/bundling.ts @@ -108,6 +108,12 @@ export interface BundlingOptions { * @default - no networking options */ readonly network?: string; + + /** + * Which option to use to copy the source files to the docker container and output files back + * @default - BIND_MOUNT + */ + readonly fileCopyVariant?: BundlingFileCopyVariant; } /** @@ -151,6 +157,25 @@ export interface ILocalBundling { tryBundle(outputDir: string, options: BundlingOptions): boolean; } +/** + * The type of file copy that should be used for bundling + */ +export enum BundlingFileCopyVariant { + /** + * Creates temporary volumes and docker containers + * This is slower, but works also in more complex situations with remote or shared docker sockets. + */ + DOCKER_COPY = 'docker-copy', + + /** + * The source and output folders will be mounted as bind mount from the host system + * This is faster and simpler, but less portable than the other option. + */ + BIND_MOUNT = 'bind-mount', + +} + + /** * A Docker image used for asset bundling * diff --git a/packages/@aws-cdk/core/test/staging.test.ts b/packages/@aws-cdk/core/test/staging.test.ts index 16e14f492adaf..43434810ea7df 100644 --- a/packages/@aws-cdk/core/test/staging.test.ts +++ b/packages/@aws-cdk/core/test/staging.test.ts @@ -16,6 +16,7 @@ enum DockerStubCommand { SUCCESS_NO_OUTPUT = 'DOCKER_STUB_SUCCESS_NO_OUTPUT', MULTIPLE_FILES = 'DOCKER_STUB_MULTIPLE_FILES', SINGLE_ARCHIVE = 'DOCKER_STUB_SINGLE_ARCHIVE', + VOLUME_SINGLE_ARCHIVE = 'DOCKER_STUB_VOLUME_SINGLE_ARCHIVE', } const FIXTURE_TEST1_DIR = path.join(__dirname, 'fs', 'fixtures', 'test1'); @@ -1159,6 +1160,39 @@ describe('staging', () => { expect(staging.isArchive).toEqual(true); }); + // vtest('bundling with docker image copy variant', () => { + // // GIVEN + // const app = new App({ context: { [cxapi.NEW_STYLE_STACK_SYNTHESIS_CONTEXT]: false } }); + // const stack = new Stack(app, 'stack'); + // const directory = path.join(__dirname, 'fs', 'fixtures', 'test1'); + + // // WHEN + // const staging = new AssetStaging(stack, 'Asset', { + // sourcePath: directory, + // bundling: { + // image: DockerImage.fromRegistry('alpine'), + // command: [DockerStubCommand.VOLUME_SINGLE_ARCHIVE], + // fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY, + // }, + // }); + + // // THEN + // const assembly = app.synth(); + // expect(fs.readdirSync(assembly.directory)).toEqual([ + // 'asset.b978cc9f3faeb46e1eefdcc1f7d606ba6e0a171c90fad671e35a105b357b9c3b', // this is the bundle dir + // 'asset.b978cc9f3faeb46e1eefdcc1f7d606ba6e0a171c90fad671e35a105b357b9c3b.zip', + // 'cdk.out', + // 'manifest.json', + // 'stack.template.json', + // 'tree.json', + // ]); + // expect(fs.readdirSync(path.join(assembly.directory, 'asset.b978cc9f3faeb46e1eefdcc1f7d606ba6e0a171c90fad671e35a105b357b9c3b'))).toEqual([ + // 'test.zip', // bundle dir with "touched" bundled output file + // ]); + // expect(staging.packaging).toEqual(FileAssetPackaging.FILE); + // expect(staging.isArchive).toEqual(true); + // }); + test('bundling that produces a single archive file with disk cache', () => { // GIVEN const TEST_OUTDIR = path.join(__dirname, 'cdk.out'); From ac327d07e08f765301098983725fb94b004462e0 Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Wed, 4 Jan 2023 14:48:47 +0100 Subject: [PATCH 06/34] add basic integration test and pass through config from python lambda --- .../aws-lambda-python/lib/bundling.ts | 10 +- .../@aws-cdk/aws-lambda-python/lib/types.ts | 8 +- .../index.js | 1052 +++++++++++++++++ .../cdk.out | 1 + ...bda-python-function-dockercopy.assets.json | 32 + ...a-python-function-dockercopy.template.json | 109 ++ .../integ.json | 13 + ...efaultTestDeployAssert9D2D26DD.assets.json | 32 + ...aultTestDeployAssert9D2D26DD.template.json | 178 +++ .../manifest.json | 160 +++ .../tree.json | 343 ++++++ .../test/integ.function.dockercopy.ts | 51 + .../test/lambda-handler-dockercopy/index.py | 8 + 13 files changed, 1995 insertions(+), 2 deletions(-) create mode 100644 packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/asset.278d42fa865f60954d898636503d0ee86a6689d73dc50eb912fac62def0ef6a4.bundle/index.js create mode 100644 packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/cdk.out create mode 100644 packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/integ-lambda-python-function-dockercopy.assets.json create mode 100644 packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/integ-lambda-python-function-dockercopy.template.json create mode 100644 packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/integ.json create mode 100644 packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/lambdapythonfunctiondockercopyDefaultTestDeployAssert9D2D26DD.assets.json create mode 100644 packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/lambdapythonfunctiondockercopyDefaultTestDeployAssert9D2D26DD.template.json create mode 100644 packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/manifest.json create mode 100644 packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/tree.json create mode 100644 packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.ts create mode 100644 packages/@aws-cdk/aws-lambda-python/test/lambda-handler-dockercopy/index.py diff --git a/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts b/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts index f83756a334fe8..e7a629907ecd1 100644 --- a/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts +++ b/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts @@ -1,6 +1,6 @@ import * as path from 'path'; import { Architecture, AssetCode, Code, Runtime } from '@aws-cdk/aws-lambda'; -import { AssetStaging, BundlingOptions as CdkBundlingOptions, DockerImage, DockerVolume } from '@aws-cdk/core'; +import { AssetStaging, BundlingFileCopyVariant, BundlingOptions as CdkBundlingOptions, DockerImage, DockerVolume } from '@aws-cdk/core'; import { Packaging, DependenciesFile } from './packaging'; import { BundlingOptions, ICommandHooks } from './types'; @@ -41,6 +41,12 @@ export interface BundlingProps extends BundlingOptions { * @default - Does not skip bundling */ readonly skip?: boolean; + + /** + * Which option to use to copy the source files to the docker container and output files back + * @default - BIND_MOUNT + */ + fileCopyVariant?: BundlingFileCopyVariant } /** @@ -66,6 +72,7 @@ export class Bundling implements CdkBundlingOptions { public readonly user?: string; public readonly securityOpt?: string; public readonly network?: string; + public readonly fileCopyVariant?: BundlingFileCopyVariant; constructor(props: BundlingProps) { const { @@ -104,6 +111,7 @@ export class Bundling implements CdkBundlingOptions { this.user = props.user; this.securityOpt = props.securityOpt; this.network = props.network; + this.fileCopyVariant = props.fileCopyVariant; } private createBundlingCommand(options: BundlingCommandOptions): string[] { diff --git a/packages/@aws-cdk/aws-lambda-python/lib/types.ts b/packages/@aws-cdk/aws-lambda-python/lib/types.ts index ad0ff6f8ce09d..625a232d82396 100644 --- a/packages/@aws-cdk/aws-lambda-python/lib/types.ts +++ b/packages/@aws-cdk/aws-lambda-python/lib/types.ts @@ -1,4 +1,4 @@ -import { AssetHashType, DockerImage, DockerRunOptions } from '@aws-cdk/core'; +import { AssetHashType, BundlingFileCopyVariant, DockerImage, DockerRunOptions } from '@aws-cdk/core'; /** @@ -86,6 +86,12 @@ export interface BundlingOptions extends DockerRunOptions { * @default - do not run additional commands */ readonly commandHooks?: ICommandHooks; + + /** + * Which option to use to copy the source files to the docker container and output files back + * @default - BIND_MOUNT + */ + readonly fileCopyVariant?: BundlingFileCopyVariant; } /** diff --git a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/asset.278d42fa865f60954d898636503d0ee86a6689d73dc50eb912fac62def0ef6a4.bundle/index.js b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/asset.278d42fa865f60954d898636503d0ee86a6689d73dc50eb912fac62def0ef6a4.bundle/index.js new file mode 100644 index 0000000000000..2bf09d6726a42 --- /dev/null +++ b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/asset.278d42fa865f60954d898636503d0ee86a6689d73dc50eb912fac62def0ef6a4.bundle/index.js @@ -0,0 +1,1052 @@ +"use strict"; +var __create = Object.create; +var __defProp = Object.defineProperty; +var __getOwnPropDesc = Object.getOwnPropertyDescriptor; +var __getOwnPropNames = Object.getOwnPropertyNames; +var __getProtoOf = Object.getPrototypeOf; +var __hasOwnProp = Object.prototype.hasOwnProperty; +var __export = (target, all) => { + for (var name in all) + __defProp(target, name, { get: all[name], enumerable: true }); +}; +var __copyProps = (to, from, except, desc) => { + if (from && typeof from === "object" || typeof from === "function") { + for (let key of __getOwnPropNames(from)) + if (!__hasOwnProp.call(to, key) && key !== except) + __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable }); + } + return to; +}; +var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps( + isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target, + mod +)); +var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod); + +// lib/assertions/providers/lambda-handler/index.ts +var lambda_handler_exports = {}; +__export(lambda_handler_exports, { + handler: () => handler, + isComplete: () => isComplete, + onTimeout: () => onTimeout +}); +module.exports = __toCommonJS(lambda_handler_exports); + +// ../assertions/lib/matcher.ts +var Matcher = class { + static isMatcher(x) { + return x && x instanceof Matcher; + } +}; +var MatchResult = class { + constructor(target) { + this.failuresHere = /* @__PURE__ */ new Map(); + this.captures = /* @__PURE__ */ new Map(); + this.finalized = false; + this.innerMatchFailures = /* @__PURE__ */ new Map(); + this._hasFailed = false; + this._failCount = 0; + this._cost = 0; + this.target = target; + } + push(matcher, path, message) { + return this.recordFailure({ matcher, path, message }); + } + recordFailure(failure) { + const failKey = failure.path.join("."); + let list = this.failuresHere.get(failKey); + if (!list) { + list = []; + this.failuresHere.set(failKey, list); + } + this._failCount += 1; + this._cost += failure.cost ?? 1; + list.push(failure); + this._hasFailed = true; + return this; + } + get isSuccess() { + return !this._hasFailed; + } + hasFailed() { + return this._hasFailed; + } + get failCount() { + return this._failCount; + } + get failCost() { + return this._cost; + } + compose(id, inner) { + if (inner.hasFailed()) { + this._hasFailed = true; + this._failCount += inner.failCount; + this._cost += inner._cost; + this.innerMatchFailures.set(id, inner); + } + inner.captures.forEach((vals, capture) => { + vals.forEach((value) => this.recordCapture({ capture, value })); + }); + return this; + } + finished() { + if (this.finalized) { + return this; + } + if (this.failCount === 0) { + this.captures.forEach((vals, cap) => cap._captured.push(...vals)); + } + this.finalized = true; + return this; + } + toHumanStrings() { + const failures = new Array(); + debugger; + recurse(this, []); + return failures.map((r) => { + const loc = r.path.length === 0 ? "" : ` at /${r.path.join("/")}`; + return "" + r.message + loc + ` (using ${r.matcher.name} matcher)`; + }); + function recurse(x, prefix) { + for (const fail of Array.from(x.failuresHere.values()).flat()) { + failures.push({ + matcher: fail.matcher, + message: fail.message, + path: [...prefix, ...fail.path] + }); + } + for (const [key, inner] of x.innerMatchFailures.entries()) { + recurse(inner, [...prefix, key]); + } + } + } + renderMismatch() { + if (!this.hasFailed()) { + return ""; + } + const parts = new Array(); + const indents = new Array(); + emitFailures(this, ""); + recurse(this); + return moveMarkersToFront(parts.join("").trimEnd()); + function emit(x) { + if (x === void 0) { + debugger; + } + parts.push(x.replace(/\n/g, ` +${indents.join("")}`)); + } + function emitFailures(r, path, scrapSet) { + for (const fail of r.failuresHere.get(path) ?? []) { + emit(`!! ${fail.message} +`); + } + scrapSet == null ? void 0 : scrapSet.delete(path); + } + function recurse(r) { + const remainingFailures = new Set(Array.from(r.failuresHere.keys()).filter((x) => x !== "")); + if (Array.isArray(r.target)) { + indents.push(" "); + emit("[\n"); + for (const [first, i] of enumFirst(range(r.target.length))) { + if (!first) { + emit(",\n"); + } + emitFailures(r, `${i}`, remainingFailures); + const innerMatcher = r.innerMatchFailures.get(`${i}`); + if (innerMatcher) { + emitFailures(innerMatcher, ""); + recurseComparingValues(innerMatcher, r.target[i]); + } else { + emit(renderAbridged(r.target[i])); + } + } + emitRemaining(); + indents.pop(); + emit("\n]"); + return; + } + if (r.target && typeof r.target === "object") { + indents.push(" "); + emit("{\n"); + const keys = Array.from(/* @__PURE__ */ new Set([ + ...Object.keys(r.target), + ...Array.from(remainingFailures) + ])).sort(); + for (const [first, key] of enumFirst(keys)) { + if (!first) { + emit(",\n"); + } + emitFailures(r, key, remainingFailures); + const innerMatcher = r.innerMatchFailures.get(key); + if (innerMatcher) { + emitFailures(innerMatcher, ""); + emit(`${jsonify(key)}: `); + recurseComparingValues(innerMatcher, r.target[key]); + } else { + emit(`${jsonify(key)}: `); + emit(renderAbridged(r.target[key])); + } + } + emitRemaining(); + indents.pop(); + emit("\n}"); + return; + } + emitRemaining(); + emit(jsonify(r.target)); + function emitRemaining() { + if (remainingFailures.size > 0) { + emit("\n"); + } + for (const key of remainingFailures) { + emitFailures(r, key); + } + } + } + function recurseComparingValues(inner, actualValue) { + if (inner.target === actualValue) { + return recurse(inner); + } + emit(renderAbridged(actualValue)); + emit(" <*> "); + recurse(inner); + } + function renderAbridged(x) { + if (Array.isArray(x)) { + switch (x.length) { + case 0: + return "[]"; + case 1: + return `[ ${renderAbridged(x[0])} ]`; + case 2: + if (x.every((e) => ["number", "boolean", "string"].includes(typeof e))) { + return `[ ${x.map(renderAbridged).join(", ")} ]`; + } + return "[ ... ]"; + default: + return "[ ... ]"; + } + } + if (x && typeof x === "object") { + const keys = Object.keys(x); + switch (keys.length) { + case 0: + return "{}"; + case 1: + return `{ ${JSON.stringify(keys[0])}: ${renderAbridged(x[keys[0]])} }`; + default: + return "{ ... }"; + } + } + return jsonify(x); + } + function jsonify(x) { + return JSON.stringify(x) ?? "undefined"; + } + function moveMarkersToFront(x) { + const re = /^(\s+)!!/gm; + return x.replace(re, (_, spaces) => `!!${spaces.substring(0, spaces.length - 2)}`); + } + } + recordCapture(options) { + let values = this.captures.get(options.capture); + if (values === void 0) { + values = []; + } + values.push(options.value); + this.captures.set(options.capture, values); + } +}; +function* range(n) { + for (let i = 0; i < n; i++) { + yield i; + } +} +function* enumFirst(xs) { + let first = true; + for (const x of xs) { + yield [first, x]; + first = false; + } +} + +// ../assertions/lib/private/matchers/absent.ts +var AbsentMatch = class extends Matcher { + constructor(name) { + super(); + this.name = name; + } + test(actual) { + const result = new MatchResult(actual); + if (actual !== void 0) { + result.recordFailure({ + matcher: this, + path: [], + message: `Received ${actual}, but key should be absent` + }); + } + return result; + } +}; + +// ../assertions/lib/private/sorting.ts +function sortKeyComparator(keyFn) { + return (a, b) => { + const ak = keyFn(a); + const bk = keyFn(b); + for (let i = 0; i < ak.length && i < bk.length; i++) { + const av = ak[i]; + const bv = bk[i]; + let diff = 0; + if (typeof av === "number" && typeof bv === "number") { + diff = av - bv; + } else if (typeof av === "string" && typeof bv === "string") { + diff = av.localeCompare(bv); + } + if (diff !== 0) { + return diff; + } + } + return bk.length - ak.length; + }; +} + +// ../assertions/lib/private/sparse-matrix.ts +var SparseMatrix = class { + constructor() { + this.matrix = /* @__PURE__ */ new Map(); + } + get(row, col) { + var _a; + return (_a = this.matrix.get(row)) == null ? void 0 : _a.get(col); + } + row(row) { + var _a; + return Array.from(((_a = this.matrix.get(row)) == null ? void 0 : _a.entries()) ?? []); + } + set(row, col, value) { + let r = this.matrix.get(row); + if (!r) { + r = /* @__PURE__ */ new Map(); + this.matrix.set(row, r); + } + r.set(col, value); + } +}; + +// ../assertions/lib/private/type.ts +function getType(obj) { + return Array.isArray(obj) ? "array" : typeof obj; +} + +// ../assertions/lib/match.ts +var Match = class { + static absent() { + return new AbsentMatch("absent"); + } + static arrayWith(pattern) { + return new ArrayMatch("arrayWith", pattern); + } + static arrayEquals(pattern) { + return new ArrayMatch("arrayEquals", pattern, { subsequence: false }); + } + static exact(pattern) { + return new LiteralMatch("exact", pattern, { partialObjects: false }); + } + static objectLike(pattern) { + return new ObjectMatch("objectLike", pattern); + } + static objectEquals(pattern) { + return new ObjectMatch("objectEquals", pattern, { partial: false }); + } + static not(pattern) { + return new NotMatch("not", pattern); + } + static serializedJson(pattern) { + return new SerializedJson("serializedJson", pattern); + } + static anyValue() { + return new AnyMatch("anyValue"); + } + static stringLikeRegexp(pattern) { + return new StringLikeRegexpMatch("stringLikeRegexp", pattern); + } +}; +var LiteralMatch = class extends Matcher { + constructor(name, pattern, options = {}) { + super(); + this.name = name; + this.pattern = pattern; + this.partialObjects = options.partialObjects ?? false; + if (Matcher.isMatcher(this.pattern)) { + throw new Error("LiteralMatch cannot directly contain another matcher. Remove the top-level matcher or nest it more deeply."); + } + } + test(actual) { + if (Array.isArray(this.pattern)) { + return new ArrayMatch(this.name, this.pattern, { subsequence: false, partialObjects: this.partialObjects }).test(actual); + } + if (typeof this.pattern === "object") { + return new ObjectMatch(this.name, this.pattern, { partial: this.partialObjects }).test(actual); + } + const result = new MatchResult(actual); + if (typeof this.pattern !== typeof actual) { + result.recordFailure({ + matcher: this, + path: [], + message: `Expected type ${typeof this.pattern} but received ${getType(actual)}` + }); + return result; + } + if (actual !== this.pattern) { + result.recordFailure({ + matcher: this, + path: [], + message: `Expected ${this.pattern} but received ${actual}` + }); + } + return result; + } +}; +var ArrayMatch = class extends Matcher { + constructor(name, pattern, options = {}) { + super(); + this.name = name; + this.pattern = pattern; + this.subsequence = options.subsequence ?? true; + this.partialObjects = options.partialObjects ?? false; + } + test(actual) { + if (!Array.isArray(actual)) { + return new MatchResult(actual).recordFailure({ + matcher: this, + path: [], + message: `Expected type array but received ${getType(actual)}` + }); + } + return this.subsequence ? this.testSubsequence(actual) : this.testFullArray(actual); + } + testFullArray(actual) { + const result = new MatchResult(actual); + let i = 0; + for (; i < this.pattern.length && i < actual.length; i++) { + const patternElement = this.pattern[i]; + const matcher = Matcher.isMatcher(patternElement) ? patternElement : new LiteralMatch(this.name, patternElement, { partialObjects: this.partialObjects }); + const innerResult = matcher.test(actual[i]); + result.compose(`${i}`, innerResult); + } + if (i < this.pattern.length) { + result.recordFailure({ + matcher: this, + message: `Not enough elements in array (expecting ${this.pattern.length}, got ${actual.length})`, + path: [`${i}`] + }); + } + if (i < actual.length) { + result.recordFailure({ + matcher: this, + message: `Too many elements in array (expecting ${this.pattern.length}, got ${actual.length})`, + path: [`${i}`] + }); + } + return result; + } + testSubsequence(actual) { + const result = new MatchResult(actual); + let patternIdx = 0; + let actualIdx = 0; + const matches = new SparseMatrix(); + while (patternIdx < this.pattern.length && actualIdx < actual.length) { + const patternElement = this.pattern[patternIdx]; + const matcher = Matcher.isMatcher(patternElement) ? patternElement : new LiteralMatch(this.name, patternElement, { partialObjects: this.partialObjects }); + const matcherName = matcher.name; + if (matcherName == "absent" || matcherName == "anyValue") { + throw new Error(`The Matcher ${matcherName}() cannot be nested within arrayWith()`); + } + const innerResult = matcher.test(actual[actualIdx]); + matches.set(patternIdx, actualIdx, innerResult); + actualIdx++; + if (innerResult.isSuccess) { + result.compose(`${actualIdx}`, innerResult); + patternIdx++; + } + } + if (patternIdx < this.pattern.length) { + for (let spi = 0; spi < patternIdx; spi++) { + const foundMatch = matches.row(spi).find(([, r]) => r.isSuccess); + if (!foundMatch) { + continue; + } + const [index] = foundMatch; + result.compose(`${index}`, new MatchResult(actual[index]).recordFailure({ + matcher: this, + message: `arrayWith pattern ${spi} matched here`, + path: [], + cost: 0 + })); + } + const failedMatches = matches.row(patternIdx); + failedMatches.sort(sortKeyComparator(([i, r]) => [r.failCost, i])); + if (failedMatches.length > 0) { + const [index, innerResult] = failedMatches[0]; + result.recordFailure({ + matcher: this, + message: `Could not match arrayWith pattern ${patternIdx}. This is the closest match`, + path: [`${index}`], + cost: 0 + }); + result.compose(`${index}`, innerResult); + } else { + result.recordFailure({ + matcher: this, + message: `Could not match arrayWith pattern ${patternIdx}. No more elements to try`, + path: [`${actual.length}`] + }); + } + } + return result; + } +}; +var ObjectMatch = class extends Matcher { + constructor(name, pattern, options = {}) { + super(); + this.name = name; + this.pattern = pattern; + this.partial = options.partial ?? true; + } + test(actual) { + if (typeof actual !== "object" || Array.isArray(actual)) { + return new MatchResult(actual).recordFailure({ + matcher: this, + path: [], + message: `Expected type object but received ${getType(actual)}` + }); + } + const result = new MatchResult(actual); + if (!this.partial) { + for (const a of Object.keys(actual)) { + if (!(a in this.pattern)) { + result.recordFailure({ + matcher: this, + path: [a], + message: `Unexpected key ${a}` + }); + } + } + } + for (const [patternKey, patternVal] of Object.entries(this.pattern)) { + if (!(patternKey in actual) && !(patternVal instanceof AbsentMatch)) { + result.recordFailure({ + matcher: this, + path: [patternKey], + message: `Missing key '${patternKey}'` + }); + continue; + } + const matcher = Matcher.isMatcher(patternVal) ? patternVal : new LiteralMatch(this.name, patternVal, { partialObjects: this.partial }); + const inner = matcher.test(actual[patternKey]); + result.compose(patternKey, inner); + } + return result; + } +}; +var SerializedJson = class extends Matcher { + constructor(name, pattern) { + super(); + this.name = name; + this.pattern = pattern; + } + test(actual) { + if (getType(actual) !== "string") { + return new MatchResult(actual).recordFailure({ + matcher: this, + path: [], + message: `Expected JSON as a string but found ${getType(actual)}` + }); + } + let parsed; + try { + parsed = JSON.parse(actual); + } catch (err) { + if (err instanceof SyntaxError) { + return new MatchResult(actual).recordFailure({ + matcher: this, + path: [], + message: `Invalid JSON string: ${actual}` + }); + } else { + throw err; + } + } + const matcher = Matcher.isMatcher(this.pattern) ? this.pattern : new LiteralMatch(this.name, this.pattern); + const innerResult = matcher.test(parsed); + if (innerResult.hasFailed()) { + innerResult.recordFailure({ + matcher: this, + path: [], + message: "Encoded JSON value does not match" + }); + } + return innerResult; + } +}; +var NotMatch = class extends Matcher { + constructor(name, pattern) { + super(); + this.name = name; + this.pattern = pattern; + } + test(actual) { + const matcher = Matcher.isMatcher(this.pattern) ? this.pattern : new LiteralMatch(this.name, this.pattern); + const innerResult = matcher.test(actual); + const result = new MatchResult(actual); + if (innerResult.failCount === 0) { + result.recordFailure({ + matcher: this, + path: [], + message: `Found unexpected match: ${JSON.stringify(actual, void 0, 2)}` + }); + } + return result; + } +}; +var AnyMatch = class extends Matcher { + constructor(name) { + super(); + this.name = name; + } + test(actual) { + const result = new MatchResult(actual); + if (actual == null) { + result.recordFailure({ + matcher: this, + path: [], + message: "Expected a value but found none" + }); + } + return result; + } +}; +var StringLikeRegexpMatch = class extends Matcher { + constructor(name, pattern) { + super(); + this.name = name; + this.pattern = pattern; + } + test(actual) { + const result = new MatchResult(actual); + const regex = new RegExp(this.pattern, "gm"); + if (typeof actual !== "string") { + result.recordFailure({ + matcher: this, + path: [], + message: `Expected a string, but got '${typeof actual}'` + }); + } + if (!regex.test(actual)) { + result.recordFailure({ + matcher: this, + path: [], + message: `String '${actual}' did not match pattern '${this.pattern}'` + }); + } + return result; + } +}; + +// lib/assertions/providers/lambda-handler/base.ts +var https = __toESM(require("https")); +var url = __toESM(require("url")); +var AWS = __toESM(require("aws-sdk")); +var CustomResourceHandler = class { + constructor(event, context) { + this.event = event; + this.context = context; + this.timedOut = false; + this.timeout = setTimeout(async () => { + await this.respond({ + status: "FAILED", + reason: "Lambda Function Timeout", + data: this.context.logStreamName + }); + this.timedOut = true; + }, context.getRemainingTimeInMillis() - 1200); + this.event = event; + this.physicalResourceId = extractPhysicalResourceId(event); + } + async handle() { + try { + if ("stateMachineArn" in this.event.ResourceProperties) { + const req = { + stateMachineArn: this.event.ResourceProperties.stateMachineArn, + name: this.event.RequestId, + input: JSON.stringify(this.event) + }; + await this.startExecution(req); + return; + } else { + const response = await this.processEvent(this.event.ResourceProperties); + return response; + } + } catch (e) { + console.log(e); + throw e; + } finally { + clearTimeout(this.timeout); + } + } + async handleIsComplete() { + try { + const result = await this.processEvent(this.event.ResourceProperties); + return result; + } catch (e) { + console.log(e); + return; + } finally { + clearTimeout(this.timeout); + } + } + async startExecution(req) { + try { + const sfn = new AWS.StepFunctions(); + await sfn.startExecution(req).promise(); + } finally { + clearTimeout(this.timeout); + } + } + respond(response) { + if (this.timedOut) { + return; + } + const cfResponse = { + Status: response.status, + Reason: response.reason, + PhysicalResourceId: this.physicalResourceId, + StackId: this.event.StackId, + RequestId: this.event.RequestId, + LogicalResourceId: this.event.LogicalResourceId, + NoEcho: false, + Data: response.data + }; + const responseBody = JSON.stringify(cfResponse); + console.log("Responding to CloudFormation", responseBody); + const parsedUrl = url.parse(this.event.ResponseURL); + const requestOptions = { + hostname: parsedUrl.hostname, + path: parsedUrl.path, + method: "PUT", + headers: { "content-type": "", "content-length": responseBody.length } + }; + return new Promise((resolve, reject) => { + try { + const request2 = https.request(requestOptions, resolve); + request2.on("error", reject); + request2.write(responseBody); + request2.end(); + } catch (e) { + reject(e); + } finally { + clearTimeout(this.timeout); + } + }); + } +}; +function extractPhysicalResourceId(event) { + switch (event.RequestType) { + case "Create": + return event.LogicalResourceId; + case "Update": + case "Delete": + return event.PhysicalResourceId; + } +} + +// lib/assertions/providers/lambda-handler/assertion.ts +var AssertionHandler = class extends CustomResourceHandler { + async processEvent(request2) { + let actual = decodeCall(request2.actual); + const expected = decodeCall(request2.expected); + let result; + const matcher = new MatchCreator(expected).getMatcher(); + console.log(`Testing equality between ${JSON.stringify(request2.actual)} and ${JSON.stringify(request2.expected)}`); + const matchResult = matcher.test(actual); + matchResult.finished(); + if (matchResult.hasFailed()) { + result = { + failed: true, + assertion: JSON.stringify({ + status: "fail", + message: matchResult.renderMismatch() + }) + }; + if (request2.failDeployment) { + throw new Error(result.assertion); + } + } else { + result = { + assertion: JSON.stringify({ + status: "success" + }) + }; + } + return result; + } +}; +var MatchCreator = class { + constructor(obj) { + this.parsedObj = { + matcher: obj + }; + } + getMatcher() { + try { + const final = JSON.parse(JSON.stringify(this.parsedObj), function(_k, v) { + const nested = Object.keys(v)[0]; + switch (nested) { + case "$ArrayWith": + return Match.arrayWith(v[nested]); + case "$ObjectLike": + return Match.objectLike(v[nested]); + case "$StringLike": + return Match.stringLikeRegexp(v[nested]); + case "$SerializedJson": + return Match.serializedJson(v[nested]); + default: + return v; + } + }); + if (Matcher.isMatcher(final.matcher)) { + return final.matcher; + } + return Match.exact(final.matcher); + } catch { + return Match.exact(this.parsedObj.matcher); + } + } +}; +function decodeCall(call) { + if (!call) { + return void 0; + } + try { + const parsed = JSON.parse(call); + return parsed; + } catch (e) { + return call; + } +} + +// lib/assertions/providers/lambda-handler/utils.ts +function decode(object) { + return JSON.parse(JSON.stringify(object), (_k, v) => { + switch (v) { + case "TRUE:BOOLEAN": + return true; + case "FALSE:BOOLEAN": + return false; + default: + return v; + } + }); +} + +// lib/assertions/providers/lambda-handler/sdk.ts +function flatten(object) { + return Object.assign( + {}, + ...function _flatten(child, path = []) { + return [].concat(...Object.keys(child).map((key) => { + let childKey = Buffer.isBuffer(child[key]) ? child[key].toString("utf8") : child[key]; + if (typeof childKey === "string") { + childKey = isJsonString(childKey); + } + return typeof childKey === "object" && childKey !== null ? _flatten(childKey, path.concat([key])) : { [path.concat([key]).join(".")]: childKey }; + })); + }(object) + ); +} +var AwsApiCallHandler = class extends CustomResourceHandler { + async processEvent(request2) { + const AWS2 = require("aws-sdk"); + console.log(`AWS SDK VERSION: ${AWS2.VERSION}`); + if (!Object.prototype.hasOwnProperty.call(AWS2, request2.service)) { + throw Error(`Service ${request2.service} does not exist in AWS SDK version ${AWS2.VERSION}.`); + } + const service = new AWS2[request2.service](); + const response = await service[request2.api](request2.parameters && decode(request2.parameters)).promise(); + console.log(`SDK response received ${JSON.stringify(response)}`); + delete response.ResponseMetadata; + const respond = { + apiCallResponse: response + }; + const flatData = { + ...flatten(respond) + }; + let resp = respond; + if (request2.outputPaths) { + resp = filterKeys(flatData, request2.outputPaths); + } else if (request2.flattenResponse === "true") { + resp = flatData; + } + console.log(`Returning result ${JSON.stringify(resp)}`); + return resp; + } +}; +function filterKeys(object, searchStrings) { + return Object.entries(object).reduce((filteredObject, [key, value]) => { + for (const searchString of searchStrings) { + if (key.startsWith(`apiCallResponse.${searchString}`)) { + filteredObject[key] = value; + } + } + return filteredObject; + }, {}); +} +function isJsonString(value) { + try { + return JSON.parse(value); + } catch { + return value; + } +} + +// lib/assertions/providers/lambda-handler/types.ts +var ASSERT_RESOURCE_TYPE = "Custom::DeployAssert@AssertEquals"; +var SDK_RESOURCE_TYPE_PREFIX = "Custom::DeployAssert@SdkCall"; + +// lib/assertions/providers/lambda-handler/index.ts +async function handler(event, context) { + console.log(`Event: ${JSON.stringify({ ...event, ResponseURL: "..." })}`); + const provider = createResourceHandler(event, context); + try { + if (event.RequestType === "Delete") { + await provider.respond({ + status: "SUCCESS", + reason: "OK" + }); + return; + } + const result = await provider.handle(); + if ("stateMachineArn" in event.ResourceProperties) { + console.info('Found "stateMachineArn", waiter statemachine started'); + return; + } else if ("expected" in event.ResourceProperties) { + console.info('Found "expected", testing assertions'); + const actualPath = event.ResourceProperties.actualPath; + const actual = actualPath ? result[`apiCallResponse.${actualPath}`] : result.apiCallResponse; + const assertion = new AssertionHandler({ + ...event, + ResourceProperties: { + ServiceToken: event.ServiceToken, + actual, + expected: event.ResourceProperties.expected + } + }, context); + try { + const assertionResult = await assertion.handle(); + await provider.respond({ + status: "SUCCESS", + reason: "OK", + data: { + ...assertionResult, + ...result + } + }); + return; + } catch (e) { + await provider.respond({ + status: "FAILED", + reason: e.message ?? "Internal Error" + }); + return; + } + } + await provider.respond({ + status: "SUCCESS", + reason: "OK", + data: result + }); + } catch (e) { + await provider.respond({ + status: "FAILED", + reason: e.message ?? "Internal Error" + }); + return; + } + return; +} +async function onTimeout(timeoutEvent) { + const isCompleteRequest = JSON.parse(JSON.parse(timeoutEvent.Cause).errorMessage); + const provider = createResourceHandler(isCompleteRequest, standardContext); + await provider.respond({ + status: "FAILED", + reason: "Operation timed out: " + JSON.stringify(isCompleteRequest) + }); +} +async function isComplete(event, context) { + console.log(`Event: ${JSON.stringify({ ...event, ResponseURL: "..." })}`); + const provider = createResourceHandler(event, context); + try { + const result = await provider.handleIsComplete(); + const actualPath = event.ResourceProperties.actualPath; + if (result) { + const actual = actualPath ? result[`apiCallResponse.${actualPath}`] : result.apiCallResponse; + if ("expected" in event.ResourceProperties) { + const assertion = new AssertionHandler({ + ...event, + ResourceProperties: { + ServiceToken: event.ServiceToken, + actual, + expected: event.ResourceProperties.expected + } + }, context); + const assertionResult = await assertion.handleIsComplete(); + if (!(assertionResult == null ? void 0 : assertionResult.failed)) { + await provider.respond({ + status: "SUCCESS", + reason: "OK", + data: { + ...assertionResult, + ...result + } + }); + return; + } else { + console.log(`Assertion Failed: ${JSON.stringify(assertionResult)}`); + throw new Error(JSON.stringify(event)); + } + } + await provider.respond({ + status: "SUCCESS", + reason: "OK", + data: result + }); + } else { + console.log("No result"); + throw new Error(JSON.stringify(event)); + } + return; + } catch (e) { + console.log(e); + throw new Error(JSON.stringify(event)); + } +} +function createResourceHandler(event, context) { + if (event.ResourceType.startsWith(SDK_RESOURCE_TYPE_PREFIX)) { + return new AwsApiCallHandler(event, context); + } else if (event.ResourceType.startsWith(ASSERT_RESOURCE_TYPE)) { + return new AssertionHandler(event, context); + } else { + throw new Error(`Unsupported resource type "${event.ResourceType}`); + } +} +var standardContext = { + getRemainingTimeInMillis: () => 9e4 +}; +// Annotate the CommonJS export names for ESM import in node: +0 && (module.exports = { + handler, + isComplete, + onTimeout +}); diff --git a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/cdk.out b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/cdk.out new file mode 100644 index 0000000000000..145739f539580 --- /dev/null +++ b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/cdk.out @@ -0,0 +1 @@ +{"version":"22.0.0"} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/integ-lambda-python-function-dockercopy.assets.json b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/integ-lambda-python-function-dockercopy.assets.json new file mode 100644 index 0000000000000..f028996eb2625 --- /dev/null +++ b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/integ-lambda-python-function-dockercopy.assets.json @@ -0,0 +1,32 @@ +{ + "version": "22.0.0", + "files": { + "39e85bcb459167e07de90088cda39e0e2722b0642639a97e9dc32517666af2c1": { + "source": { + "path": "asset.39e85bcb459167e07de90088cda39e0e2722b0642639a97e9dc32517666af2c1", + "packaging": "zip" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "39e85bcb459167e07de90088cda39e0e2722b0642639a97e9dc32517666af2c1.zip", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + }, + "bd0eb483c090553c87ab388a80d805311afdbf9a6f99d6f48311df4fc6a38edc": { + "source": { + "path": "integ-lambda-python-function-dockercopy.template.json", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "bd0eb483c090553c87ab388a80d805311afdbf9a6f99d6f48311df4fc6a38edc.json", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + } + }, + "dockerImages": {} +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/integ-lambda-python-function-dockercopy.template.json b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/integ-lambda-python-function-dockercopy.template.json new file mode 100644 index 0000000000000..7ee7c8481e579 --- /dev/null +++ b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/integ-lambda-python-function-dockercopy.template.json @@ -0,0 +1,109 @@ +{ + "Resources": { + "myhandlerServiceRole77891068": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + ] + } + ] + } + }, + "myhandlerD202FA8E": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "S3Key": "39e85bcb459167e07de90088cda39e0e2722b0642639a97e9dc32517666af2c1.zip" + }, + "Role": { + "Fn::GetAtt": [ + "myhandlerServiceRole77891068", + "Arn" + ] + }, + "Handler": "index.handler", + "Runtime": "python3.9" + }, + "DependsOn": [ + "myhandlerServiceRole77891068" + ] + } + }, + "Outputs": { + "FunctionArn": { + "Value": { + "Fn::GetAtt": [ + "myhandlerD202FA8E", + "Arn" + ] + } + }, + "ExportsOutputRefmyhandlerD202FA8E369E8804": { + "Value": { + "Ref": "myhandlerD202FA8E" + }, + "Export": { + "Name": "integ-lambda-python-function-dockercopy:ExportsOutputRefmyhandlerD202FA8E369E8804" + } + } + }, + "Parameters": { + "BootstrapVersion": { + "Type": "AWS::SSM::Parameter::Value", + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]" + } + }, + "Rules": { + "CheckBootstrapVersion": { + "Assertions": [ + { + "Assert": { + "Fn::Not": [ + { + "Fn::Contains": [ + [ + "1", + "2", + "3", + "4", + "5" + ], + { + "Ref": "BootstrapVersion" + } + ] + } + ] + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." + } + ] + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/integ.json b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/integ.json new file mode 100644 index 0000000000000..a38a97f7dcfd0 --- /dev/null +++ b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/integ.json @@ -0,0 +1,13 @@ +{ + "version": "22.0.0", + "testCases": { + "lambda-python-function-dockercopy/DefaultTest": { + "stacks": [ + "integ-lambda-python-function-dockercopy" + ], + "stackUpdateWorkflow": false, + "assertionStack": "lambda-python-function-dockercopy/DefaultTest/DeployAssert", + "assertionStackName": "lambdapythonfunctiondockercopyDefaultTestDeployAssert9D2D26DD" + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/lambdapythonfunctiondockercopyDefaultTestDeployAssert9D2D26DD.assets.json b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/lambdapythonfunctiondockercopyDefaultTestDeployAssert9D2D26DD.assets.json new file mode 100644 index 0000000000000..d3916f944ac22 --- /dev/null +++ b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/lambdapythonfunctiondockercopyDefaultTestDeployAssert9D2D26DD.assets.json @@ -0,0 +1,32 @@ +{ + "version": "22.0.0", + "files": { + "278d42fa865f60954d898636503d0ee86a6689d73dc50eb912fac62def0ef6a4": { + "source": { + "path": "asset.278d42fa865f60954d898636503d0ee86a6689d73dc50eb912fac62def0ef6a4.bundle", + "packaging": "zip" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "278d42fa865f60954d898636503d0ee86a6689d73dc50eb912fac62def0ef6a4.zip", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + }, + "ad7d0233f02af46f2b6f92d91509f304be9609e052ba07f17ac0a025d4fb718c": { + "source": { + "path": "lambdapythonfunctiondockercopyDefaultTestDeployAssert9D2D26DD.template.json", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "ad7d0233f02af46f2b6f92d91509f304be9609e052ba07f17ac0a025d4fb718c.json", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + } + }, + "dockerImages": {} +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/lambdapythonfunctiondockercopyDefaultTestDeployAssert9D2D26DD.template.json b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/lambdapythonfunctiondockercopyDefaultTestDeployAssert9D2D26DD.template.json new file mode 100644 index 0000000000000..e437648a82b69 --- /dev/null +++ b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/lambdapythonfunctiondockercopyDefaultTestDeployAssert9D2D26DD.template.json @@ -0,0 +1,178 @@ +{ + "Resources": { + "LambdaInvoke431773224924ebf10c8a31d363a6bf16": { + "Type": "Custom::DeployAssert@SdkCallLambdainvoke", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "SingletonFunction1488541a7b23466481b69b4408076b81HandlerCD40AE9F", + "Arn" + ] + }, + "service": "Lambda", + "api": "invoke", + "expected": "{\"$ObjectLike\":{\"Payload\":\"200\"}}", + "parameters": { + "FunctionName": { + "Fn::ImportValue": "integ-lambda-python-function-dockercopy:ExportsOutputRefmyhandlerD202FA8E369E8804" + } + }, + "flattenResponse": "false", + "salt": "1672828354944" + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "LambdaInvoke431773224924ebf10c8a31d363a6bf16Invoke9BC0E67F": { + "Type": "AWS::Lambda::Permission", + "Properties": { + "Action": "lambda:InvokeFunction", + "FunctionName": { + "Fn::ImportValue": "integ-lambda-python-function-dockercopy:ExportsOutputRefmyhandlerD202FA8E369E8804" + }, + "Principal": { + "Fn::GetAtt": [ + "SingletonFunction1488541a7b23466481b69b4408076b81Role37ABCE73", + "Arn" + ] + } + } + }, + "SingletonFunction1488541a7b23466481b69b4408076b81Role37ABCE73": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ] + }, + "ManagedPolicyArns": [ + { + "Fn::Sub": "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + } + ], + "Policies": [ + { + "PolicyName": "Inline", + "PolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "lambda:Invoke" + ], + "Effect": "Allow", + "Resource": [ + "*" + ] + }, + { + "Action": [ + "lambda:InvokeFunction" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":lambda:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":function:", + { + "Fn::ImportValue": "integ-lambda-python-function-dockercopy:ExportsOutputRefmyhandlerD202FA8E369E8804" + } + ] + ] + } + ] + } + ] + } + } + ] + } + }, + "SingletonFunction1488541a7b23466481b69b4408076b81HandlerCD40AE9F": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Runtime": "nodejs14.x", + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "S3Key": "278d42fa865f60954d898636503d0ee86a6689d73dc50eb912fac62def0ef6a4.zip" + }, + "Timeout": 120, + "Handler": "index.handler", + "Role": { + "Fn::GetAtt": [ + "SingletonFunction1488541a7b23466481b69b4408076b81Role37ABCE73", + "Arn" + ] + } + } + } + }, + "Outputs": { + "AssertionResultsLambdaInvoke431773224924ebf10c8a31d363a6bf16": { + "Value": { + "Fn::GetAtt": [ + "LambdaInvoke431773224924ebf10c8a31d363a6bf16", + "assertion" + ] + } + } + }, + "Parameters": { + "BootstrapVersion": { + "Type": "AWS::SSM::Parameter::Value", + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]" + } + }, + "Rules": { + "CheckBootstrapVersion": { + "Assertions": [ + { + "Assert": { + "Fn::Not": [ + { + "Fn::Contains": [ + [ + "1", + "2", + "3", + "4", + "5" + ], + { + "Ref": "BootstrapVersion" + } + ] + } + ] + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." + } + ] + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/manifest.json b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/manifest.json new file mode 100644 index 0000000000000..7a75420e87cb0 --- /dev/null +++ b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/manifest.json @@ -0,0 +1,160 @@ +{ + "version": "22.0.0", + "artifacts": { + "integ-lambda-python-function-dockercopy.assets": { + "type": "cdk:asset-manifest", + "properties": { + "file": "integ-lambda-python-function-dockercopy.assets.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "integ-lambda-python-function-dockercopy": { + "type": "aws:cloudformation:stack", + "environment": "aws://unknown-account/unknown-region", + "properties": { + "templateFile": "integ-lambda-python-function-dockercopy.template.json", + "validateOnSynth": false, + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", + "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/bd0eb483c090553c87ab388a80d805311afdbf9a6f99d6f48311df4fc6a38edc.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", + "additionalDependencies": [ + "integ-lambda-python-function-dockercopy.assets" + ], + "lookupRole": { + "arn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-lookup-role-${AWS::AccountId}-${AWS::Region}", + "requiresBootstrapStackVersion": 8, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "dependencies": [ + "integ-lambda-python-function-dockercopy.assets" + ], + "metadata": { + "/integ-lambda-python-function-dockercopy/my_handler/ServiceRole/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "myhandlerServiceRole77891068" + } + ], + "/integ-lambda-python-function-dockercopy/my_handler/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "myhandlerD202FA8E" + } + ], + "/integ-lambda-python-function-dockercopy/FunctionArn": [ + { + "type": "aws:cdk:logicalId", + "data": "FunctionArn" + } + ], + "/integ-lambda-python-function-dockercopy/Exports/Output{\"Ref\":\"myhandlerD202FA8E\"}": [ + { + "type": "aws:cdk:logicalId", + "data": "ExportsOutputRefmyhandlerD202FA8E369E8804" + } + ], + "/integ-lambda-python-function-dockercopy/BootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "BootstrapVersion" + } + ], + "/integ-lambda-python-function-dockercopy/CheckBootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "CheckBootstrapVersion" + } + ] + }, + "displayName": "integ-lambda-python-function-dockercopy" + }, + "lambdapythonfunctiondockercopyDefaultTestDeployAssert9D2D26DD.assets": { + "type": "cdk:asset-manifest", + "properties": { + "file": "lambdapythonfunctiondockercopyDefaultTestDeployAssert9D2D26DD.assets.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "lambdapythonfunctiondockercopyDefaultTestDeployAssert9D2D26DD": { + "type": "aws:cloudformation:stack", + "environment": "aws://unknown-account/unknown-region", + "properties": { + "templateFile": "lambdapythonfunctiondockercopyDefaultTestDeployAssert9D2D26DD.template.json", + "validateOnSynth": false, + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", + "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/ad7d0233f02af46f2b6f92d91509f304be9609e052ba07f17ac0a025d4fb718c.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", + "additionalDependencies": [ + "lambdapythonfunctiondockercopyDefaultTestDeployAssert9D2D26DD.assets" + ], + "lookupRole": { + "arn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-lookup-role-${AWS::AccountId}-${AWS::Region}", + "requiresBootstrapStackVersion": 8, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "dependencies": [ + "integ-lambda-python-function-dockercopy", + "lambdapythonfunctiondockercopyDefaultTestDeployAssert9D2D26DD.assets" + ], + "metadata": { + "/lambda-python-function-dockercopy/DefaultTest/DeployAssert/LambdaInvoke431773224924ebf10c8a31d363a6bf16/Default/Default": [ + { + "type": "aws:cdk:logicalId", + "data": "LambdaInvoke431773224924ebf10c8a31d363a6bf16" + } + ], + "/lambda-python-function-dockercopy/DefaultTest/DeployAssert/LambdaInvoke431773224924ebf10c8a31d363a6bf16/Invoke": [ + { + "type": "aws:cdk:logicalId", + "data": "LambdaInvoke431773224924ebf10c8a31d363a6bf16Invoke9BC0E67F" + } + ], + "/lambda-python-function-dockercopy/DefaultTest/DeployAssert/LambdaInvoke431773224924ebf10c8a31d363a6bf16/AssertionResults": [ + { + "type": "aws:cdk:logicalId", + "data": "AssertionResultsLambdaInvoke431773224924ebf10c8a31d363a6bf16" + } + ], + "/lambda-python-function-dockercopy/DefaultTest/DeployAssert/SingletonFunction1488541a7b23466481b69b4408076b81/Role": [ + { + "type": "aws:cdk:logicalId", + "data": "SingletonFunction1488541a7b23466481b69b4408076b81Role37ABCE73" + } + ], + "/lambda-python-function-dockercopy/DefaultTest/DeployAssert/SingletonFunction1488541a7b23466481b69b4408076b81/Handler": [ + { + "type": "aws:cdk:logicalId", + "data": "SingletonFunction1488541a7b23466481b69b4408076b81HandlerCD40AE9F" + } + ], + "/lambda-python-function-dockercopy/DefaultTest/DeployAssert/BootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "BootstrapVersion" + } + ], + "/lambda-python-function-dockercopy/DefaultTest/DeployAssert/CheckBootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "CheckBootstrapVersion" + } + ] + }, + "displayName": "lambda-python-function-dockercopy/DefaultTest/DeployAssert" + }, + "Tree": { + "type": "cdk:tree", + "properties": { + "file": "tree.json" + } + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/tree.json b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/tree.json new file mode 100644 index 0000000000000..a29b4014bb06f --- /dev/null +++ b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/tree.json @@ -0,0 +1,343 @@ +{ + "version": "tree-0.1", + "tree": { + "id": "App", + "path": "", + "children": { + "integ-lambda-python-function-dockercopy": { + "id": "integ-lambda-python-function-dockercopy", + "path": "integ-lambda-python-function-dockercopy", + "children": { + "my_handler": { + "id": "my_handler", + "path": "integ-lambda-python-function-dockercopy/my_handler", + "children": { + "ServiceRole": { + "id": "ServiceRole", + "path": "integ-lambda-python-function-dockercopy/my_handler/ServiceRole", + "children": { + "ImportServiceRole": { + "id": "ImportServiceRole", + "path": "integ-lambda-python-function-dockercopy/my_handler/ServiceRole/ImportServiceRole", + "constructInfo": { + "fqn": "@aws-cdk/core.Resource", + "version": "2.59.0-rc.0" + } + }, + "Resource": { + "id": "Resource", + "path": "integ-lambda-python-function-dockercopy/my_handler/ServiceRole/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Role", + "aws:cdk:cloudformation:props": { + "assumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "managedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + ] + } + ] + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-iam.CfnRole", + "version": "2.59.0-rc.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-iam.Role", + "version": "2.59.0-rc.0" + } + }, + "Code": { + "id": "Code", + "path": "integ-lambda-python-function-dockercopy/my_handler/Code", + "children": { + "Stage": { + "id": "Stage", + "path": "integ-lambda-python-function-dockercopy/my_handler/Code/Stage", + "constructInfo": { + "fqn": "@aws-cdk/core.AssetStaging", + "version": "2.59.0-rc.0" + } + }, + "AssetBucket": { + "id": "AssetBucket", + "path": "integ-lambda-python-function-dockercopy/my_handler/Code/AssetBucket", + "constructInfo": { + "fqn": "@aws-cdk/aws-s3.BucketBase", + "version": "2.59.0-rc.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-s3-assets.Asset", + "version": "2.59.0-rc.0" + } + }, + "Resource": { + "id": "Resource", + "path": "integ-lambda-python-function-dockercopy/my_handler/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Lambda::Function", + "aws:cdk:cloudformation:props": { + "code": { + "s3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "s3Key": "39e85bcb459167e07de90088cda39e0e2722b0642639a97e9dc32517666af2c1.zip" + }, + "role": { + "Fn::GetAtt": [ + "myhandlerServiceRole77891068", + "Arn" + ] + }, + "handler": "index.handler", + "runtime": "python3.9" + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-lambda.CfnFunction", + "version": "2.59.0-rc.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-lambda-python.PythonFunction", + "version": "2.59.0-rc.0" + } + }, + "FunctionArn": { + "id": "FunctionArn", + "path": "integ-lambda-python-function-dockercopy/FunctionArn", + "constructInfo": { + "fqn": "@aws-cdk/core.CfnOutput", + "version": "2.59.0-rc.0" + } + }, + "Exports": { + "id": "Exports", + "path": "integ-lambda-python-function-dockercopy/Exports", + "children": { + "Output{\"Ref\":\"myhandlerD202FA8E\"}": { + "id": "Output{\"Ref\":\"myhandlerD202FA8E\"}", + "path": "integ-lambda-python-function-dockercopy/Exports/Output{\"Ref\":\"myhandlerD202FA8E\"}", + "constructInfo": { + "fqn": "@aws-cdk/core.CfnOutput", + "version": "2.59.0-rc.0" + } + } + }, + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.1.189" + } + }, + "BootstrapVersion": { + "id": "BootstrapVersion", + "path": "integ-lambda-python-function-dockercopy/BootstrapVersion", + "constructInfo": { + "fqn": "@aws-cdk/core.CfnParameter", + "version": "2.59.0-rc.0" + } + }, + "CheckBootstrapVersion": { + "id": "CheckBootstrapVersion", + "path": "integ-lambda-python-function-dockercopy/CheckBootstrapVersion", + "constructInfo": { + "fqn": "@aws-cdk/core.CfnRule", + "version": "2.59.0-rc.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/core.Stack", + "version": "2.59.0-rc.0" + } + }, + "lambda-python-function-dockercopy": { + "id": "lambda-python-function-dockercopy", + "path": "lambda-python-function-dockercopy", + "children": { + "DefaultTest": { + "id": "DefaultTest", + "path": "lambda-python-function-dockercopy/DefaultTest", + "children": { + "Default": { + "id": "Default", + "path": "lambda-python-function-dockercopy/DefaultTest/Default", + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.1.189" + } + }, + "DeployAssert": { + "id": "DeployAssert", + "path": "lambda-python-function-dockercopy/DefaultTest/DeployAssert", + "children": { + "LambdaInvoke431773224924ebf10c8a31d363a6bf16": { + "id": "LambdaInvoke431773224924ebf10c8a31d363a6bf16", + "path": "lambda-python-function-dockercopy/DefaultTest/DeployAssert/LambdaInvoke431773224924ebf10c8a31d363a6bf16", + "children": { + "SdkProvider": { + "id": "SdkProvider", + "path": "lambda-python-function-dockercopy/DefaultTest/DeployAssert/LambdaInvoke431773224924ebf10c8a31d363a6bf16/SdkProvider", + "children": { + "AssertionsProvider": { + "id": "AssertionsProvider", + "path": "lambda-python-function-dockercopy/DefaultTest/DeployAssert/LambdaInvoke431773224924ebf10c8a31d363a6bf16/SdkProvider/AssertionsProvider", + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.1.189" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/integ-tests.AssertionsProvider", + "version": "2.59.0-rc.0" + } + }, + "Default": { + "id": "Default", + "path": "lambda-python-function-dockercopy/DefaultTest/DeployAssert/LambdaInvoke431773224924ebf10c8a31d363a6bf16/Default", + "children": { + "Default": { + "id": "Default", + "path": "lambda-python-function-dockercopy/DefaultTest/DeployAssert/LambdaInvoke431773224924ebf10c8a31d363a6bf16/Default/Default", + "constructInfo": { + "fqn": "@aws-cdk/core.CfnResource", + "version": "2.59.0-rc.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/core.CustomResource", + "version": "2.59.0-rc.0" + } + }, + "Invoke": { + "id": "Invoke", + "path": "lambda-python-function-dockercopy/DefaultTest/DeployAssert/LambdaInvoke431773224924ebf10c8a31d363a6bf16/Invoke", + "constructInfo": { + "fqn": "@aws-cdk/core.CfnResource", + "version": "2.59.0-rc.0" + } + }, + "AssertionResults": { + "id": "AssertionResults", + "path": "lambda-python-function-dockercopy/DefaultTest/DeployAssert/LambdaInvoke431773224924ebf10c8a31d363a6bf16/AssertionResults", + "constructInfo": { + "fqn": "@aws-cdk/core.CfnOutput", + "version": "2.59.0-rc.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/integ-tests.LambdaInvokeFunction", + "version": "2.59.0-rc.0" + } + }, + "SingletonFunction1488541a7b23466481b69b4408076b81": { + "id": "SingletonFunction1488541a7b23466481b69b4408076b81", + "path": "lambda-python-function-dockercopy/DefaultTest/DeployAssert/SingletonFunction1488541a7b23466481b69b4408076b81", + "children": { + "Staging": { + "id": "Staging", + "path": "lambda-python-function-dockercopy/DefaultTest/DeployAssert/SingletonFunction1488541a7b23466481b69b4408076b81/Staging", + "constructInfo": { + "fqn": "@aws-cdk/core.AssetStaging", + "version": "2.59.0-rc.0" + } + }, + "Role": { + "id": "Role", + "path": "lambda-python-function-dockercopy/DefaultTest/DeployAssert/SingletonFunction1488541a7b23466481b69b4408076b81/Role", + "constructInfo": { + "fqn": "@aws-cdk/core.CfnResource", + "version": "2.59.0-rc.0" + } + }, + "Handler": { + "id": "Handler", + "path": "lambda-python-function-dockercopy/DefaultTest/DeployAssert/SingletonFunction1488541a7b23466481b69b4408076b81/Handler", + "constructInfo": { + "fqn": "@aws-cdk/core.CfnResource", + "version": "2.59.0-rc.0" + } + } + }, + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.1.189" + } + }, + "BootstrapVersion": { + "id": "BootstrapVersion", + "path": "lambda-python-function-dockercopy/DefaultTest/DeployAssert/BootstrapVersion", + "constructInfo": { + "fqn": "@aws-cdk/core.CfnParameter", + "version": "2.59.0-rc.0" + } + }, + "CheckBootstrapVersion": { + "id": "CheckBootstrapVersion", + "path": "lambda-python-function-dockercopy/DefaultTest/DeployAssert/CheckBootstrapVersion", + "constructInfo": { + "fqn": "@aws-cdk/core.CfnRule", + "version": "2.59.0-rc.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/core.Stack", + "version": "2.59.0-rc.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/integ-tests.IntegTestCase", + "version": "2.59.0-rc.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/integ-tests.IntegTest", + "version": "2.59.0-rc.0" + } + }, + "Tree": { + "id": "Tree", + "path": "Tree", + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.1.189" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/core.App", + "version": "2.59.0-rc.0" + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.ts b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.ts new file mode 100644 index 0000000000000..200f3ffa33a1b --- /dev/null +++ b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.ts @@ -0,0 +1,51 @@ +// disabling update workflow because we don't want to include the assets in the snapshot +// python bundling changes the asset hash pretty frequently +/// !cdk-integ pragma:disable-update-workflow +import * as path from 'path'; +import { Runtime } from '@aws-cdk/aws-lambda'; +import { App, CfnOutput, Stack, StackProps, BundlingFileCopyVariant } from '@aws-cdk/core'; +import { IntegTest, ExpectedResult } from '@aws-cdk/integ-tests'; +import { Construct } from 'constructs'; +import * as lambda from '../lib'; + +/* + * Stack verification steps: + * * aws lambda invoke --function-name --invocation-type Event --payload '"OK"' response.json + */ + +class TestStack extends Stack { + public readonly functionName: string; + constructor(scope: Construct, id: string, props?: StackProps) { + super(scope, id, props); + + const fn = new lambda.PythonFunction(this, 'my_handler', { + entry: path.join(__dirname, 'lambda-handler-dockercopy'), + runtime: Runtime.PYTHON_3_9, + bundling: { + fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY, + }, + }); + this.functionName = fn.functionName; + + new CfnOutput(this, 'FunctionArn', { + value: fn.functionArn, + }); + } +} + +const app = new App(); +const testCase = new TestStack(app, 'integ-lambda-python-function-dockercopy'); +const integ = new IntegTest(app, 'lambda-python-function-dockercopy', { + testCases: [testCase], + stackUpdateWorkflow: false, +}); + +const invoke = integ.assertions.invokeFunction({ + functionName: testCase.functionName, +}); + +invoke.expect(ExpectedResult.objectLike({ + Payload: '200', +})); + +app.synth(); diff --git a/packages/@aws-cdk/aws-lambda-python/test/lambda-handler-dockercopy/index.py b/packages/@aws-cdk/aws-lambda-python/test/lambda-handler-dockercopy/index.py new file mode 100644 index 0000000000000..04f99eb108b30 --- /dev/null +++ b/packages/@aws-cdk/aws-lambda-python/test/lambda-handler-dockercopy/index.py @@ -0,0 +1,8 @@ +import requests + +def handler(event, context): + response = requests.get('https://a0.awsstatic.com/main/images/logos/aws_smile-header-desktop-en-white_59x35.png', stream=True) + + print(response.status_code) + + return response.status_code From fd367ef84476069d8a657ea875cba462df7155a5 Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Thu, 5 Jan 2023 08:35:35 +0000 Subject: [PATCH 07/34] move to own class and cleanup --- packages/@aws-cdk/core/lib/asset-staging.ts | 34 +++------ packages/@aws-cdk/core/lib/bundling.ts | 85 ++++++++++++++++++++- 2 files changed, 93 insertions(+), 26 deletions(-) diff --git a/packages/@aws-cdk/core/lib/asset-staging.ts b/packages/@aws-cdk/core/lib/asset-staging.ts index e39facfb36f25..fce9bc15e0a4f 100644 --- a/packages/@aws-cdk/core/lib/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/asset-staging.ts @@ -5,7 +5,7 @@ import * as cxapi from '@aws-cdk/cx-api'; import { Construct } from 'constructs'; import * as fs from 'fs-extra'; import { AssetHashType, AssetOptions, FileAssetPackaging } from './assets'; -import { BundlingFileCopyVariant, BundlingOptions, BundlingOutput, dockerExec, DockerVolume } from './bundling'; +import { BundlingFileCopyVariant, BundlingOptions, BundlingOutput, DockerImageBundlingCopyHelper, DockerVolume } from './bundling'; import { FileSystem, FingerprintOptions } from './fs'; import { clearLargeFileFingerprintCache } from './fs/fingerprint'; import { Names } from './names'; @@ -466,29 +466,14 @@ export class AssetStaging extends Construct { : '1000:1000'; } - const copySuffix = crypto.randomBytes(12).toString('hex'); - const inputVolumeName = `assetInput${copySuffix}`; - const outputVolumeName = `assetOutput${copySuffix}`; - const copyContainerName = `copy${copySuffix}`; - let volumesFrom: string[] = options.volumesFrom ?? []; + const helperContainer = new DockerImageBundlingCopyHelper(); if (options.fileCopyVariant?.valueOf() === BundlingFileCopyVariant.DOCKER_COPY.valueOf()) { - volumesFrom = [copyContainerName, ...options.volumesFrom ?? []]; - dockerExec(['volume', 'create', inputVolumeName]); - dockerExec(['volume', 'create', outputVolumeName]); - - dockerExec([ - 'run', - '--name', copyContainerName, - '-v', `${inputVolumeName}:${AssetStaging.BUNDLING_INPUT_DIR}`, - '-v', `${outputVolumeName}:${AssetStaging.BUNDLING_OUTPUT_DIR}`, - 'alpine', - 'sh', - '-c', - `mkdir -p ${AssetStaging.BUNDLING_INPUT_DIR} && chown -R ${user} ${AssetStaging.BUNDLING_OUTPUT_DIR} && chown -R ${user} ${AssetStaging.BUNDLING_INPUT_DIR}`, - ]); - dockerExec(['cp', `${this.sourcePath}/.`, `${copyContainerName}:${AssetStaging.BUNDLING_INPUT_DIR}`]); + volumesFrom = [helperContainer.copyContainerName, ...options.volumesFrom ?? []]; + helperContainer.prepareVolumes(); + helperContainer.startHelperContainer(user); + helperContainer.copyInputFrom(this.sourcePath); } options.image.run({ @@ -503,10 +488,9 @@ export class AssetStaging extends Construct { }); if (options.fileCopyVariant?.valueOf() === BundlingFileCopyVariant.DOCKER_COPY.valueOf()) { - dockerExec(['cp', `${copyContainerName}:${AssetStaging.BUNDLING_OUTPUT_DIR}/.`, bundleDir]); - dockerExec(['rm', copyContainerName]); - dockerExec(['volume', 'rm', inputVolumeName]); - dockerExec(['volume', 'rm', outputVolumeName]); + helperContainer.copyOutputTo(bundleDir); + helperContainer.cleanHelperContainer(); + helperContainer.cleanVolumes(); } } } catch (err) { diff --git a/packages/@aws-cdk/core/lib/bundling.ts b/packages/@aws-cdk/core/lib/bundling.ts index 553a8834e1b05..6b0c64c3c7052 100644 --- a/packages/@aws-cdk/core/lib/bundling.ts +++ b/packages/@aws-cdk/core/lib/bundling.ts @@ -1,6 +1,7 @@ import { spawnSync, SpawnSyncOptions } from 'child_process'; import * as crypto from 'crypto'; import { isAbsolute, join } from 'path'; +import { AssetStaging } from './asset-staging'; import { FileSystem } from './fs'; import { quiet, reset } from './private/jsii-deprecated'; @@ -558,11 +559,93 @@ export interface DockerBuildOptions { readonly targetStage?: string; } +/** + * Provides a helper container for copying bundling related files to specific input and output volumes + */ +export class DockerImageBundlingCopyHelper { + /** + * Name of the Docker volume that is used for the asset input + */ + private inputVolumeName: string; + /** + * Name of the Docker volume that is used for the asset output + */ + private outputVolumeName: string; + /** + * Name of the Docker helper container to copy files into the volume + */ + public copyContainerName: string; + + constructor() { + const copySuffix = crypto.randomBytes(12).toString('hex'); + this.inputVolumeName = `assetInput${copySuffix}`; + this.outputVolumeName = `assetOutput${copySuffix}`; + this.copyContainerName = `copy${copySuffix}`; + } + + /** + * Creates volumes for asset input and output + */ + public prepareVolumes() { + dockerExec(['volume', 'create', this.inputVolumeName]); + dockerExec(['volume', 'create', this.outputVolumeName]); + } + + /** + * Removes volumes for asset input and output + */ + public cleanVolumes() { + dockerExec(['volume', 'rm', this.inputVolumeName]); + dockerExec(['volume', 'rm', this.outputVolumeName]); + } + + /** + * runs a helper container that holds volumes and does some preparation tasks + * @param user The user that will later access these files and needs permissions to do so + */ + public startHelperContainer(user: string) { + dockerExec([ + 'run', + '--name', this.copyContainerName, + '-v', `${this.inputVolumeName}:${AssetStaging.BUNDLING_INPUT_DIR}`, + '-v', `${this.outputVolumeName}:${AssetStaging.BUNDLING_OUTPUT_DIR}`, + 'alpine', + 'sh', + '-c', + `mkdir -p ${AssetStaging.BUNDLING_INPUT_DIR} && chown -R ${user} ${AssetStaging.BUNDLING_OUTPUT_DIR} && chown -R ${user} ${AssetStaging.BUNDLING_INPUT_DIR}`, + ]); + } + + /** + * removes the Docker helper container + */ + public cleanHelperContainer() { + dockerExec(['rm', this.copyContainerName]); + } + + /** + * copy files from the host where this is executed into the input volume + * @param sourcePath - path to folder where files should be copied from - without trailing slash + */ + public copyInputFrom(sourcePath: string) { + dockerExec(['cp', `${sourcePath}/.`, `${this.copyContainerName}:${AssetStaging.BUNDLING_INPUT_DIR}`]); + + } + + /** + * copy files from the the output volume to the host where this is executed + * @param outputPath - path to folder where files should be copied to - without trailing slash + */ + public copyOutputTo(outputPath: string) { + dockerExec(['cp', `${this.copyContainerName}:${AssetStaging.BUNDLING_OUTPUT_DIR}/.`, outputPath]); + } +} + function flatten(x: string[][]) { return Array.prototype.concat([], ...x); } -export function dockerExec(args: string[], options?: SpawnSyncOptions) { +function dockerExec(args: string[], options?: SpawnSyncOptions) { const prog = process.env.CDK_DOCKER ?? 'docker'; const proc = spawnSync(prog, args, options ?? { stdio: [ // show Docker output From 26c1d100998c2d9f4a37e7f323c4698221a66ea8 Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Thu, 5 Jan 2023 15:58:32 +0100 Subject: [PATCH 08/34] add tests --- .../aws-lambda-python/test/bundling.test.ts | 18 ++- ...bda-python-function-dockercopy.assets.json | 10 +- ...a-python-function-dockercopy.template.json | 2 +- ...efaultTestDeployAssert9D2D26DD.assets.json | 4 +- ...aultTestDeployAssert9D2D26DD.template.json | 2 +- .../manifest.json | 4 +- .../tree.json | 58 +++---- .../lambda-handler-dockercopy/.ignorefile | 0 .../test/lambda-handler-dockercopy/Pipfile | 7 + .../lambda-handler-dockercopy/Pipfile.lock | 59 ++++++++ packages/@aws-cdk/core/lib/asset-staging.ts | 32 ++-- packages/@aws-cdk/core/lib/bundling.ts | 2 +- packages/@aws-cdk/core/test/bundling.test.ts | 143 +++++++++++++++++- packages/@aws-cdk/core/test/docker-stub-cp.sh | 16 ++ packages/@aws-cdk/core/test/staging.test.ts | 113 +++++++++----- 15 files changed, 373 insertions(+), 97 deletions(-) create mode 100644 packages/@aws-cdk/aws-lambda-python/test/lambda-handler-dockercopy/.ignorefile create mode 100644 packages/@aws-cdk/aws-lambda-python/test/lambda-handler-dockercopy/Pipfile create mode 100644 packages/@aws-cdk/aws-lambda-python/test/lambda-handler-dockercopy/Pipfile.lock create mode 100755 packages/@aws-cdk/core/test/docker-stub-cp.sh diff --git a/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts b/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts index d6172ccfb705b..f3b40596d47a7 100644 --- a/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts +++ b/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts @@ -1,7 +1,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { Architecture, Code, Runtime } from '@aws-cdk/aws-lambda'; -import { DockerImage } from '@aws-cdk/core'; +import { BundlingFileCopyVariant, DockerImage } from '@aws-cdk/core'; import { Bundling } from '../lib/bundling'; jest.spyOn(Code, 'fromAsset'); @@ -374,6 +374,22 @@ test('Bundling with custom network', () => { })); }); +test('Bundling with docker copy variant', () => { + const entry = path.join(__dirname, 'lambda-handler'); + Bundling.bundle({ + entry: entry, + runtime: Runtime.PYTHON_3_7, + fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY, + + }); + + expect(Code.fromAsset).toHaveBeenCalledWith(entry, expect.objectContaining({ + bundling: expect.objectContaining({ + fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY, + }), + })); +}); + test('Do not build docker image when skipping bundling', () => { const entry = path.join(__dirname, 'lambda-handler'); Bundling.bundle({ diff --git a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/integ-lambda-python-function-dockercopy.assets.json b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/integ-lambda-python-function-dockercopy.assets.json index f028996eb2625..a5f2dee381914 100644 --- a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/integ-lambda-python-function-dockercopy.assets.json +++ b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/integ-lambda-python-function-dockercopy.assets.json @@ -1,20 +1,20 @@ { "version": "22.0.0", "files": { - "39e85bcb459167e07de90088cda39e0e2722b0642639a97e9dc32517666af2c1": { + "d7f71e188167c705d230ed88d3e1935d888bb138e142570c4c22139b79f23aa9": { "source": { - "path": "asset.39e85bcb459167e07de90088cda39e0e2722b0642639a97e9dc32517666af2c1", + "path": "asset.d7f71e188167c705d230ed88d3e1935d888bb138e142570c4c22139b79f23aa9", "packaging": "zip" }, "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "39e85bcb459167e07de90088cda39e0e2722b0642639a97e9dc32517666af2c1.zip", + "objectKey": "d7f71e188167c705d230ed88d3e1935d888bb138e142570c4c22139b79f23aa9.zip", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } }, - "bd0eb483c090553c87ab388a80d805311afdbf9a6f99d6f48311df4fc6a38edc": { + "e358cdb3c29cca92c45125b79de88c4f42224ba81d32defb08b97b631d8f55e3": { "source": { "path": "integ-lambda-python-function-dockercopy.template.json", "packaging": "file" @@ -22,7 +22,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "bd0eb483c090553c87ab388a80d805311afdbf9a6f99d6f48311df4fc6a38edc.json", + "objectKey": "e358cdb3c29cca92c45125b79de88c4f42224ba81d32defb08b97b631d8f55e3.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/integ-lambda-python-function-dockercopy.template.json b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/integ-lambda-python-function-dockercopy.template.json index 7ee7c8481e579..8498a0dfcef8f 100644 --- a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/integ-lambda-python-function-dockercopy.template.json +++ b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/integ-lambda-python-function-dockercopy.template.json @@ -38,7 +38,7 @@ "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" }, - "S3Key": "39e85bcb459167e07de90088cda39e0e2722b0642639a97e9dc32517666af2c1.zip" + "S3Key": "d7f71e188167c705d230ed88d3e1935d888bb138e142570c4c22139b79f23aa9.zip" }, "Role": { "Fn::GetAtt": [ diff --git a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/lambdapythonfunctiondockercopyDefaultTestDeployAssert9D2D26DD.assets.json b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/lambdapythonfunctiondockercopyDefaultTestDeployAssert9D2D26DD.assets.json index d3916f944ac22..f9cf05dad8e75 100644 --- a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/lambdapythonfunctiondockercopyDefaultTestDeployAssert9D2D26DD.assets.json +++ b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/lambdapythonfunctiondockercopyDefaultTestDeployAssert9D2D26DD.assets.json @@ -14,7 +14,7 @@ } } }, - "ad7d0233f02af46f2b6f92d91509f304be9609e052ba07f17ac0a025d4fb718c": { + "3e42550e52b2738ac1d185ed45788d5ca72cd1c7ce5402d23c4c1097c53826c3": { "source": { "path": "lambdapythonfunctiondockercopyDefaultTestDeployAssert9D2D26DD.template.json", "packaging": "file" @@ -22,7 +22,7 @@ "destinations": { "current_account-current_region": { "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", - "objectKey": "ad7d0233f02af46f2b6f92d91509f304be9609e052ba07f17ac0a025d4fb718c.json", + "objectKey": "3e42550e52b2738ac1d185ed45788d5ca72cd1c7ce5402d23c4c1097c53826c3.json", "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" } } diff --git a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/lambdapythonfunctiondockercopyDefaultTestDeployAssert9D2D26DD.template.json b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/lambdapythonfunctiondockercopyDefaultTestDeployAssert9D2D26DD.template.json index e437648a82b69..78a740c07125a 100644 --- a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/lambdapythonfunctiondockercopyDefaultTestDeployAssert9D2D26DD.template.json +++ b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/lambdapythonfunctiondockercopyDefaultTestDeployAssert9D2D26DD.template.json @@ -18,7 +18,7 @@ } }, "flattenResponse": "false", - "salt": "1672828354944" + "salt": "1672909861311" }, "UpdateReplacePolicy": "Delete", "DeletionPolicy": "Delete" diff --git a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/manifest.json b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/manifest.json index 7a75420e87cb0..5bca70ed12ebb 100644 --- a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/manifest.json +++ b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/manifest.json @@ -17,7 +17,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/bd0eb483c090553c87ab388a80d805311afdbf9a6f99d6f48311df4fc6a38edc.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/e358cdb3c29cca92c45125b79de88c4f42224ba81d32defb08b97b631d8f55e3.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ @@ -88,7 +88,7 @@ "validateOnSynth": false, "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", - "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/ad7d0233f02af46f2b6f92d91509f304be9609e052ba07f17ac0a025d4fb718c.json", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/3e42550e52b2738ac1d185ed45788d5ca72cd1c7ce5402d23c4c1097c53826c3.json", "requiresBootstrapStackVersion": 6, "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", "additionalDependencies": [ diff --git a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/tree.json b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/tree.json index a29b4014bb06f..da9ab28f36cea 100644 --- a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/tree.json +++ b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.js.snapshot/tree.json @@ -21,7 +21,7 @@ "path": "integ-lambda-python-function-dockercopy/my_handler/ServiceRole/ImportServiceRole", "constructInfo": { "fqn": "@aws-cdk/core.Resource", - "version": "2.59.0-rc.0" + "version": "0.0.0" } }, "Resource": { @@ -60,13 +60,13 @@ }, "constructInfo": { "fqn": "@aws-cdk/aws-iam.CfnRole", - "version": "2.59.0-rc.0" + "version": "0.0.0" } } }, "constructInfo": { "fqn": "@aws-cdk/aws-iam.Role", - "version": "2.59.0-rc.0" + "version": "0.0.0" } }, "Code": { @@ -78,7 +78,7 @@ "path": "integ-lambda-python-function-dockercopy/my_handler/Code/Stage", "constructInfo": { "fqn": "@aws-cdk/core.AssetStaging", - "version": "2.59.0-rc.0" + "version": "0.0.0" } }, "AssetBucket": { @@ -86,13 +86,13 @@ "path": "integ-lambda-python-function-dockercopy/my_handler/Code/AssetBucket", "constructInfo": { "fqn": "@aws-cdk/aws-s3.BucketBase", - "version": "2.59.0-rc.0" + "version": "0.0.0" } } }, "constructInfo": { "fqn": "@aws-cdk/aws-s3-assets.Asset", - "version": "2.59.0-rc.0" + "version": "0.0.0" } }, "Resource": { @@ -105,7 +105,7 @@ "s3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" }, - "s3Key": "39e85bcb459167e07de90088cda39e0e2722b0642639a97e9dc32517666af2c1.zip" + "s3Key": "d7f71e188167c705d230ed88d3e1935d888bb138e142570c4c22139b79f23aa9.zip" }, "role": { "Fn::GetAtt": [ @@ -119,13 +119,13 @@ }, "constructInfo": { "fqn": "@aws-cdk/aws-lambda.CfnFunction", - "version": "2.59.0-rc.0" + "version": "0.0.0" } } }, "constructInfo": { "fqn": "@aws-cdk/aws-lambda-python.PythonFunction", - "version": "2.59.0-rc.0" + "version": "0.0.0" } }, "FunctionArn": { @@ -133,7 +133,7 @@ "path": "integ-lambda-python-function-dockercopy/FunctionArn", "constructInfo": { "fqn": "@aws-cdk/core.CfnOutput", - "version": "2.59.0-rc.0" + "version": "0.0.0" } }, "Exports": { @@ -145,7 +145,7 @@ "path": "integ-lambda-python-function-dockercopy/Exports/Output{\"Ref\":\"myhandlerD202FA8E\"}", "constructInfo": { "fqn": "@aws-cdk/core.CfnOutput", - "version": "2.59.0-rc.0" + "version": "0.0.0" } } }, @@ -159,7 +159,7 @@ "path": "integ-lambda-python-function-dockercopy/BootstrapVersion", "constructInfo": { "fqn": "@aws-cdk/core.CfnParameter", - "version": "2.59.0-rc.0" + "version": "0.0.0" } }, "CheckBootstrapVersion": { @@ -167,13 +167,13 @@ "path": "integ-lambda-python-function-dockercopy/CheckBootstrapVersion", "constructInfo": { "fqn": "@aws-cdk/core.CfnRule", - "version": "2.59.0-rc.0" + "version": "0.0.0" } } }, "constructInfo": { "fqn": "@aws-cdk/core.Stack", - "version": "2.59.0-rc.0" + "version": "0.0.0" } }, "lambda-python-function-dockercopy": { @@ -215,7 +215,7 @@ }, "constructInfo": { "fqn": "@aws-cdk/integ-tests.AssertionsProvider", - "version": "2.59.0-rc.0" + "version": "0.0.0" } }, "Default": { @@ -227,13 +227,13 @@ "path": "lambda-python-function-dockercopy/DefaultTest/DeployAssert/LambdaInvoke431773224924ebf10c8a31d363a6bf16/Default/Default", "constructInfo": { "fqn": "@aws-cdk/core.CfnResource", - "version": "2.59.0-rc.0" + "version": "0.0.0" } } }, "constructInfo": { "fqn": "@aws-cdk/core.CustomResource", - "version": "2.59.0-rc.0" + "version": "0.0.0" } }, "Invoke": { @@ -241,7 +241,7 @@ "path": "lambda-python-function-dockercopy/DefaultTest/DeployAssert/LambdaInvoke431773224924ebf10c8a31d363a6bf16/Invoke", "constructInfo": { "fqn": "@aws-cdk/core.CfnResource", - "version": "2.59.0-rc.0" + "version": "0.0.0" } }, "AssertionResults": { @@ -249,13 +249,13 @@ "path": "lambda-python-function-dockercopy/DefaultTest/DeployAssert/LambdaInvoke431773224924ebf10c8a31d363a6bf16/AssertionResults", "constructInfo": { "fqn": "@aws-cdk/core.CfnOutput", - "version": "2.59.0-rc.0" + "version": "0.0.0" } } }, "constructInfo": { "fqn": "@aws-cdk/integ-tests.LambdaInvokeFunction", - "version": "2.59.0-rc.0" + "version": "0.0.0" } }, "SingletonFunction1488541a7b23466481b69b4408076b81": { @@ -267,7 +267,7 @@ "path": "lambda-python-function-dockercopy/DefaultTest/DeployAssert/SingletonFunction1488541a7b23466481b69b4408076b81/Staging", "constructInfo": { "fqn": "@aws-cdk/core.AssetStaging", - "version": "2.59.0-rc.0" + "version": "0.0.0" } }, "Role": { @@ -275,7 +275,7 @@ "path": "lambda-python-function-dockercopy/DefaultTest/DeployAssert/SingletonFunction1488541a7b23466481b69b4408076b81/Role", "constructInfo": { "fqn": "@aws-cdk/core.CfnResource", - "version": "2.59.0-rc.0" + "version": "0.0.0" } }, "Handler": { @@ -283,7 +283,7 @@ "path": "lambda-python-function-dockercopy/DefaultTest/DeployAssert/SingletonFunction1488541a7b23466481b69b4408076b81/Handler", "constructInfo": { "fqn": "@aws-cdk/core.CfnResource", - "version": "2.59.0-rc.0" + "version": "0.0.0" } } }, @@ -297,7 +297,7 @@ "path": "lambda-python-function-dockercopy/DefaultTest/DeployAssert/BootstrapVersion", "constructInfo": { "fqn": "@aws-cdk/core.CfnParameter", - "version": "2.59.0-rc.0" + "version": "0.0.0" } }, "CheckBootstrapVersion": { @@ -305,25 +305,25 @@ "path": "lambda-python-function-dockercopy/DefaultTest/DeployAssert/CheckBootstrapVersion", "constructInfo": { "fqn": "@aws-cdk/core.CfnRule", - "version": "2.59.0-rc.0" + "version": "0.0.0" } } }, "constructInfo": { "fqn": "@aws-cdk/core.Stack", - "version": "2.59.0-rc.0" + "version": "0.0.0" } } }, "constructInfo": { "fqn": "@aws-cdk/integ-tests.IntegTestCase", - "version": "2.59.0-rc.0" + "version": "0.0.0" } } }, "constructInfo": { "fqn": "@aws-cdk/integ-tests.IntegTest", - "version": "2.59.0-rc.0" + "version": "0.0.0" } }, "Tree": { @@ -337,7 +337,7 @@ }, "constructInfo": { "fqn": "@aws-cdk/core.App", - "version": "2.59.0-rc.0" + "version": "0.0.0" } } } \ No newline at end of file diff --git a/packages/@aws-cdk/aws-lambda-python/test/lambda-handler-dockercopy/.ignorefile b/packages/@aws-cdk/aws-lambda-python/test/lambda-handler-dockercopy/.ignorefile new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/packages/@aws-cdk/aws-lambda-python/test/lambda-handler-dockercopy/Pipfile b/packages/@aws-cdk/aws-lambda-python/test/lambda-handler-dockercopy/Pipfile new file mode 100644 index 0000000000000..78d783bc4b9b0 --- /dev/null +++ b/packages/@aws-cdk/aws-lambda-python/test/lambda-handler-dockercopy/Pipfile @@ -0,0 +1,7 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[packages] +requests = "==2.26.0" diff --git a/packages/@aws-cdk/aws-lambda-python/test/lambda-handler-dockercopy/Pipfile.lock b/packages/@aws-cdk/aws-lambda-python/test/lambda-handler-dockercopy/Pipfile.lock new file mode 100644 index 0000000000000..1a9abf9618a62 --- /dev/null +++ b/packages/@aws-cdk/aws-lambda-python/test/lambda-handler-dockercopy/Pipfile.lock @@ -0,0 +1,59 @@ +{ + "_meta": { + "hash": { + "sha256": "6cfaa5a495be5cf47942a14b04d50e639f14743101e621684e86449dbac8da61" + }, + "pipfile-spec": 6, + "requires": {}, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "certifi": { + "hashes": [ + "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3", + "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18" + ], + "index": "pypi", + "version": "==2022.12.7" + }, + "charset-normalizer": { + "hashes": [ + "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597", + "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df" + ], + "markers": "python_version >= '3'", + "version": "==2.0.12" + }, + "idna": { + "hashes": [ + "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4", + "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2" + ], + "markers": "python_version >= '3'", + "version": "==3.4" + }, + "requests": { + "hashes": [ + "sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24", + "sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7" + ], + "index": "pypi", + "version": "==2.26.0" + }, + "urllib3": { + "hashes": [ + "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc", + "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==1.26.13" + } + }, + "develop": {} +} diff --git a/packages/@aws-cdk/core/lib/asset-staging.ts b/packages/@aws-cdk/core/lib/asset-staging.ts index fce9bc15e0a4f..de36452334948 100644 --- a/packages/@aws-cdk/core/lib/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/asset-staging.ts @@ -432,24 +432,6 @@ export class AssetStaging extends Construct { // Chmod the bundleDir to full access. fs.chmodSync(bundleDir, 0o777); - // Always mount input and output dir - let volumes: DockerVolume[] = []; - if (options.fileCopyVariant?.valueOf() === BundlingFileCopyVariant.DOCKER_COPY.valueOf()) { - volumes = options.volumes ?? []; - } else { - volumes = [ - { - hostPath: this.sourcePath, - containerPath: AssetStaging.BUNDLING_INPUT_DIR, - }, - { - hostPath: bundleDir, - containerPath: AssetStaging.BUNDLING_OUTPUT_DIR, - }, - ...options.volumes ?? [], - ]; - } - let localBundling: boolean | undefined; try { process.stderr.write(`Bundling asset ${this.node.path}...\n`); @@ -466,14 +448,28 @@ export class AssetStaging extends Construct { : '1000:1000'; } + let volumes: DockerVolume[] = []; let volumesFrom: string[] = options.volumesFrom ?? []; const helperContainer = new DockerImageBundlingCopyHelper(); if (options.fileCopyVariant?.valueOf() === BundlingFileCopyVariant.DOCKER_COPY.valueOf()) { + volumes = options.volumes ?? []; volumesFrom = [helperContainer.copyContainerName, ...options.volumesFrom ?? []]; helperContainer.prepareVolumes(); helperContainer.startHelperContainer(user); helperContainer.copyInputFrom(this.sourcePath); + } else { + volumes = [ + { + hostPath: this.sourcePath, + containerPath: AssetStaging.BUNDLING_INPUT_DIR, + }, + { + hostPath: bundleDir, + containerPath: AssetStaging.BUNDLING_OUTPUT_DIR, + }, + ...options.volumes ?? [], + ]; } options.image.run({ diff --git a/packages/@aws-cdk/core/lib/bundling.ts b/packages/@aws-cdk/core/lib/bundling.ts index 6b0c64c3c7052..3a12906ccb0fd 100644 --- a/packages/@aws-cdk/core/lib/bundling.ts +++ b/packages/@aws-cdk/core/lib/bundling.ts @@ -580,7 +580,7 @@ export class DockerImageBundlingCopyHelper { const copySuffix = crypto.randomBytes(12).toString('hex'); this.inputVolumeName = `assetInput${copySuffix}`; this.outputVolumeName = `assetOutput${copySuffix}`; - this.copyContainerName = `copy${copySuffix}`; + this.copyContainerName = `copyContainer${copySuffix}`; } /** diff --git a/packages/@aws-cdk/core/test/bundling.test.ts b/packages/@aws-cdk/core/test/bundling.test.ts index 3879d0b27fc41..1fc20efcdaf31 100644 --- a/packages/@aws-cdk/core/test/bundling.test.ts +++ b/packages/@aws-cdk/core/test/bundling.test.ts @@ -2,7 +2,7 @@ import * as child_process from 'child_process'; import * as crypto from 'crypto'; import * as path from 'path'; import * as sinon from 'sinon'; -import { DockerImage, FileSystem } from '../lib'; +import { AssetStaging, DockerImage, DockerImageBundlingCopyHelper, FileSystem } from '../lib'; describe('bundling', () => { afterEach(() => { @@ -598,4 +598,145 @@ describe('bundling', () => { 'cool', 'command', ], { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); }); + + test('DockerImageBundlingCopyHelper prepareVolumes', () => { + // GIVEN + sinon.stub(process, 'platform').value('darwin'); + const spawnSyncStub = sinon.stub(child_process, 'spawnSync').returns({ + status: 0, + stderr: Buffer.from('stderr'), + stdout: Buffer.from('stdout'), + pid: 123, + output: ['stdout', 'stderr'], + signal: null, + }); + const helper = new DockerImageBundlingCopyHelper(); + helper.prepareVolumes(); + + // THEN + expect(spawnSyncStub.calledWith('docker', sinon.match([ + 'volume', 'create', sinon.match(/assetInput.*/g), + ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); + + expect(spawnSyncStub.calledWith('docker', sinon.match([ + 'volume', 'create', sinon.match(/assetOutput.*/g), + ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); + + }); + + test('DockerImageBundlingCopyHelper cleanVolumes', () => { + // GIVEN + sinon.stub(process, 'platform').value('darwin'); + const spawnSyncStub = sinon.stub(child_process, 'spawnSync').returns({ + status: 0, + stderr: Buffer.from('stderr'), + stdout: Buffer.from('stdout'), + pid: 123, + output: ['stdout', 'stderr'], + signal: null, + }); + const helper = new DockerImageBundlingCopyHelper(); + helper.cleanVolumes(); + + // THEN + expect(spawnSyncStub.calledWith('docker', sinon.match([ + 'volume', 'rm', sinon.match(/assetInput.*/g), + ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); + + expect(spawnSyncStub.calledWith('docker', sinon.match([ + 'volume', 'rm', sinon.match(/assetOutput.*/g), + ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); + + }); + + test('DockerImageBundlingCopyHelper startHelperContainer', () => { + // GIVEN + sinon.stub(process, 'platform').value('darwin'); + const spawnSyncStub = sinon.stub(child_process, 'spawnSync').returns({ + status: 0, + stderr: Buffer.from('stderr'), + stdout: Buffer.from('stdout'), + pid: 123, + output: ['stdout', 'stderr'], + signal: null, + }); + const helper = new DockerImageBundlingCopyHelper(); + const user = '1000'; + helper.startHelperContainer(user); + + // THEN + expect(spawnSyncStub.calledWith('docker', sinon.match([ + 'run', + '--name', sinon.match(/copyContainer.*/g), + '-v', sinon.match(/assetInput.*/g), + '-v', sinon.match(/assetOutput.*/g), + 'alpine', + 'sh', + '-c', + `mkdir -p ${AssetStaging.BUNDLING_INPUT_DIR} && chown -R ${user} ${AssetStaging.BUNDLING_OUTPUT_DIR} && chown -R ${user} ${AssetStaging.BUNDLING_INPUT_DIR}`, + ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); + }); + + test('DockerImageBundlingCopyHelper cleanHelperContainer', () => { + // GIVEN + sinon.stub(process, 'platform').value('darwin'); + const spawnSyncStub = sinon.stub(child_process, 'spawnSync').returns({ + status: 0, + stderr: Buffer.from('stderr'), + stdout: Buffer.from('stdout'), + pid: 123, + output: ['stdout', 'stderr'], + signal: null, + }); + const helper = new DockerImageBundlingCopyHelper(); + helper.cleanHelperContainer(); + + // THEN + expect(spawnSyncStub.calledWith('docker', sinon.match([ + 'rm', sinon.match(/copyContainer.*/g), + ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); + }); + + test('DockerImageBundlingCopyHelper copyInputFrom', () => { + // GIVEN + sinon.stub(process, 'platform').value('darwin'); + const spawnSyncStub = sinon.stub(child_process, 'spawnSync').returns({ + status: 0, + stderr: Buffer.from('stderr'), + stdout: Buffer.from('stdout'), + pid: 123, + output: ['stdout', 'stderr'], + signal: null, + }); + const helper = new DockerImageBundlingCopyHelper(); + const dir = '/test/dir'; + helper.copyInputFrom(dir); + + // THEN + expect(spawnSyncStub.calledWith('docker', sinon.match([ + 'cp', `${dir}/.`, `${helper.copyContainerName}:${AssetStaging.BUNDLING_INPUT_DIR}`, + ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); + }); + + test('DockerImageBundlingCopyHelper copyOutputTo', () => { + // GIVEN + sinon.stub(process, 'platform').value('darwin'); + const spawnSyncStub = sinon.stub(child_process, 'spawnSync').returns({ + status: 0, + stderr: Buffer.from('stderr'), + stdout: Buffer.from('stdout'), + pid: 123, + output: ['stdout', 'stderr'], + signal: null, + }); + const helper = new DockerImageBundlingCopyHelper(); + const dir = '/test/dir'; + helper.copyOutputTo(dir); + + // THEN + expect(spawnSyncStub.calledWith('docker', sinon.match([ + 'cp', `${helper.copyContainerName}:${AssetStaging.BUNDLING_OUTPUT_DIR}/.`, dir, + ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); + }); + }); diff --git a/packages/@aws-cdk/core/test/docker-stub-cp.sh b/packages/@aws-cdk/core/test/docker-stub-cp.sh new file mode 100755 index 0000000000000..971b3185fabb3 --- /dev/null +++ b/packages/@aws-cdk/core/test/docker-stub-cp.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -euo pipefail +# stub for the `docker` executable. it is used as CDK_DOCKER when executing unit +# tests in `test.staging.ts` This variant is specific for tests that use the docker copy method for files, instead of bind mounts + +echo "$@" >> /tmp/docker-stub.input.concat +echo "$@" > /tmp/docker-stub.input + +# create a fake zip to emulate created files, fetch the target path from the "docker cp" command +if echo "$@" | grep "cp"| grep "/asset-output"; then + outdir=$(echo "$@" | grep cp | grep "/asset-output" | xargs -n1 | grep "cdk.out" | head -n1 | cut -d":" -f1) + if [ -n "$outdir" ]; then + echo "${outdir}" >> /tmp/docker-stub.input.concat.output.txt + touch "${outdir}/test.zip" + fi +fi diff --git a/packages/@aws-cdk/core/test/staging.test.ts b/packages/@aws-cdk/core/test/staging.test.ts index 43434810ea7df..001bcd13a2eee 100644 --- a/packages/@aws-cdk/core/test/staging.test.ts +++ b/packages/@aws-cdk/core/test/staging.test.ts @@ -1,3 +1,4 @@ +// import * as child_process from 'child_process'; import * as os from 'os'; import * as path from 'path'; import { testDeprecated } from '@aws-cdk/cdk-build-tools'; @@ -5,7 +6,7 @@ import { FileAssetPackaging } from '@aws-cdk/cloud-assembly-schema'; import * as cxapi from '@aws-cdk/cx-api'; import * as fs from 'fs-extra'; import * as sinon from 'sinon'; -import { App, AssetHashType, AssetStaging, DockerImage, BundlingOptions, BundlingOutput, FileSystem, Stack, Stage } from '../lib'; +import { App, AssetHashType, AssetStaging, DockerImage, BundlingOptions, BundlingOutput, FileSystem, Stack, Stage, BundlingFileCopyVariant } from '../lib'; const STUB_INPUT_FILE = '/tmp/docker-stub.input'; const STUB_INPUT_CONCAT_FILE = '/tmp/docker-stub.input.concat'; @@ -28,10 +29,16 @@ const ARCHIVE_TARBALL_TEST_HASH = '3e948ff54a277d6001e2452fdbc4a9ef61f916ff662ba const userInfo = os.userInfo(); const USER_ARG = `-u ${userInfo.uid}:${userInfo.gid}`; -// this is a way to provide a custom "docker" command for staging. -process.env.CDK_DOCKER = `${__dirname}/docker-stub.sh`; describe('staging', () => { + beforeAll(() => { + // this is a way to provide a custom "docker" command for staging. + process.env.CDK_DOCKER = `${__dirname}/docker-stub.sh`; + }); + + afterAll(() => { + delete process.env.CDK_DOCKER; + }); afterEach(() => { AssetStaging.clearAssetHashCache(); @@ -1160,39 +1167,6 @@ describe('staging', () => { expect(staging.isArchive).toEqual(true); }); - // vtest('bundling with docker image copy variant', () => { - // // GIVEN - // const app = new App({ context: { [cxapi.NEW_STYLE_STACK_SYNTHESIS_CONTEXT]: false } }); - // const stack = new Stack(app, 'stack'); - // const directory = path.join(__dirname, 'fs', 'fixtures', 'test1'); - - // // WHEN - // const staging = new AssetStaging(stack, 'Asset', { - // sourcePath: directory, - // bundling: { - // image: DockerImage.fromRegistry('alpine'), - // command: [DockerStubCommand.VOLUME_SINGLE_ARCHIVE], - // fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY, - // }, - // }); - - // // THEN - // const assembly = app.synth(); - // expect(fs.readdirSync(assembly.directory)).toEqual([ - // 'asset.b978cc9f3faeb46e1eefdcc1f7d606ba6e0a171c90fad671e35a105b357b9c3b', // this is the bundle dir - // 'asset.b978cc9f3faeb46e1eefdcc1f7d606ba6e0a171c90fad671e35a105b357b9c3b.zip', - // 'cdk.out', - // 'manifest.json', - // 'stack.template.json', - // 'tree.json', - // ]); - // expect(fs.readdirSync(path.join(assembly.directory, 'asset.b978cc9f3faeb46e1eefdcc1f7d606ba6e0a171c90fad671e35a105b357b9c3b'))).toEqual([ - // 'test.zip', // bundle dir with "touched" bundled output file - // ]); - // expect(staging.packaging).toEqual(FileAssetPackaging.FILE); - // expect(staging.isArchive).toEqual(true); - // }); - test('bundling that produces a single archive file with disk cache', () => { // GIVEN const TEST_OUTDIR = path.join(__dirname, 'cdk.out'); @@ -1286,6 +1260,73 @@ describe('staging', () => { }); }); +describe('staging with docker cp', () => { + beforeAll(() => { + // this is a way to provide a custom "docker" command for staging. + process.env.CDK_DOCKER = `${__dirname}/docker-stub-cp.sh`; + }); + + afterAll(() => { + delete process.env.CDK_DOCKER; + }); + + afterEach(() => { + AssetStaging.clearAssetHashCache(); + if (fs.existsSync(STUB_INPUT_FILE)) { + fs.unlinkSync(STUB_INPUT_FILE); + } + if (fs.existsSync(STUB_INPUT_CONCAT_FILE)) { + fs.unlinkSync(STUB_INPUT_CONCAT_FILE); + } + sinon.restore(); + }); + + test('bundling with docker image copy variant', () => { + // GIVEN + const app = new App({ context: { [cxapi.NEW_STYLE_STACK_SYNTHESIS_CONTEXT]: false } }); + const stack = new Stack(app, 'stack'); + const directory = path.join(__dirname, 'fs', 'fixtures', 'test1'); + + // WHEN + const staging = new AssetStaging(stack, 'Asset', { + sourcePath: directory, + bundling: { + image: DockerImage.fromRegistry('alpine'), + command: [DockerStubCommand.VOLUME_SINGLE_ARCHIVE], + fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY, + }, + }); + + // THEN + const assembly = app.synth(); + expect(fs.readdirSync(assembly.directory)).toEqual([ + 'asset.358c99dcb340194559aa346ecba33e7f22d3e9607bb2885623325f5eb6e0389f', // this is the bundle dir + 'asset.358c99dcb340194559aa346ecba33e7f22d3e9607bb2885623325f5eb6e0389f.zip', + 'cdk.out', + 'manifest.json', + 'stack.template.json', + 'tree.json', + ]); + expect(fs.readdirSync(path.join(assembly.directory, 'asset.358c99dcb340194559aa346ecba33e7f22d3e9607bb2885623325f5eb6e0389f'))).toEqual([ + 'test.zip', // bundle dir with "touched" bundled output file + ]); + expect(staging.packaging).toEqual(FileAssetPackaging.FILE); + expect(staging.isArchive).toEqual(true); + const dockerCalls: string[] = readDockerStubInputConcat().split(/\r?\n/); + expect(dockerCalls).toEqual(expect.arrayContaining([ + expect.stringContaining('volume create assetInput'), + expect.stringContaining('volume create assetOutput'), + expect.stringMatching('run --name copyContainer.* -v /input:/asset-input -v /output:/asset-output alpine sh -c mkdir -p /asset-input && chown -R .* /asset-output && chown -R .* /asset-input'), + expect.stringMatching('cp .*fs/fixtures/test1/\. copyContainer.*:/asset-input'), + expect.stringMatching('run --rm -u .* --volumes-from copyContainer.* -w /asset-input alpine DOCKER_STUB_VOLUME_SINGLE_ARCHIVE'), + expect.stringMatching('cp copyContainer.*:/asset-output/\. .*'), + expect.stringContaining('rm copyContainer'), + expect.stringContaining('volume rm assetInput'), + expect.stringContaining('volume rm assetOutput'), + ])); + }); +}); + // Reads a docker stub and cleans the volume paths out of the stub. function readAndCleanDockerStubInput(file: string) { return fs From 87e7a2abc55b33a39a1183dd5d13671be815c0ae Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Thu, 5 Jan 2023 16:33:57 +0100 Subject: [PATCH 09/34] add to all lambda implementations --- .../@aws-cdk/aws-lambda-go/lib/bundling.ts | 10 +++++++++ packages/@aws-cdk/aws-lambda-go/lib/types.ts | 8 ++++++- .../aws-lambda-go/test/bundling.test.ts | 20 +++++++++++++++++- .../aws-lambda-nodejs/lib/bundling.ts | 9 ++++++++ .../@aws-cdk/aws-lambda-nodejs/lib/types.ts | 8 ++++++- .../aws-lambda-nodejs/test/bundling.test.ts | 21 ++++++++++++++++++- 6 files changed, 72 insertions(+), 4 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts b/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts index 62645d0ae8cc6..3caca1a13fc05 100644 --- a/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts +++ b/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts @@ -4,6 +4,7 @@ import { Architecture, AssetCode, Code, Runtime } from '@aws-cdk/aws-lambda'; import * as cdk from '@aws-cdk/core'; import { BundlingOptions } from './types'; import { exec, findUp, getGoBuildVersion } from './util'; +import { BundlingFileCopyVariant } from '@aws-cdk/core'; /** * Options for bundling @@ -60,6 +61,12 @@ export interface BundlingProps extends BundlingOptions { * The system architecture of the lambda function */ readonly architecture: Architecture; + + /** + * Which option to use to copy the source files to the docker container and output files back + * @default - BIND_MOUNT + */ + readonly fileCopyVariant?: BundlingFileCopyVariant; } /** @@ -85,6 +92,7 @@ export class Bundling implements cdk.BundlingOptions { user: bundling.user, securityOpt: bundling.securityOpt, network: bundling.network, + fileCopyVariant: bundling.fileCopyVariant, }, }); } @@ -107,6 +115,7 @@ export class Bundling implements cdk.BundlingOptions { public readonly user?: string; public readonly securityOpt?: string; public readonly network?: string; + public readonly fileCopyVariant?: cdk.BundlingFileCopyVariant; private readonly relativeEntryPath: string; @@ -154,6 +163,7 @@ export class Bundling implements cdk.BundlingOptions { this.user = props.user; this.securityOpt = props.securityOpt; this.network = props.network; + this.fileCopyVariant = props.fileCopyVariant; // Local bundling if (!props.forcedDockerBundling) { // only if Docker is not forced diff --git a/packages/@aws-cdk/aws-lambda-go/lib/types.ts b/packages/@aws-cdk/aws-lambda-go/lib/types.ts index 5dc7beb8f886c..ec53819c7c38b 100644 --- a/packages/@aws-cdk/aws-lambda-go/lib/types.ts +++ b/packages/@aws-cdk/aws-lambda-go/lib/types.ts @@ -1,4 +1,4 @@ -import { AssetHashType, DockerImage, DockerRunOptions } from '@aws-cdk/core'; +import { AssetHashType, BundlingFileCopyVariant, DockerImage, DockerRunOptions } from '@aws-cdk/core'; /** * Bundling options @@ -111,6 +111,12 @@ export interface BundlingOptions extends DockerRunOptions { * @default - Direct access */ readonly goProxies?: string[]; + + /** + * Which option to use to copy the source files to the docker container and output files back + * @default - BIND_MOUNT + */ + readonly fileCopyVariant?: BundlingFileCopyVariant; } /** diff --git a/packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts b/packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts index 134dd0f51716f..6b6ebb0439a64 100644 --- a/packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts +++ b/packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts @@ -2,7 +2,7 @@ import * as child_process from 'child_process'; import * as os from 'os'; import * as path from 'path'; import { Architecture, Code, Runtime } from '@aws-cdk/aws-lambda'; -import { AssetHashType, DockerImage } from '@aws-cdk/core'; +import { AssetHashType, BundlingFileCopyVariant, DockerImage } from '@aws-cdk/core'; import { Bundling } from '../lib/bundling'; import * as util from '../lib/util'; @@ -461,3 +461,21 @@ test('Custom bundling network', () => { }), }); }); + +test('Custom bundling file copy variant', () => { + Bundling.bundle({ + entry, + moduleDir, + runtime: Runtime.GO_1_X, + architecture: Architecture.X86_64, + forcedDockerBundling: true, + fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY, + }); + + expect(Code.fromAsset).toHaveBeenCalledWith('/project', { + assetHashType: AssetHashType.OUTPUT, + bundling: expect.objectContaining({ + fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY, + }), + }); +}); \ No newline at end of file diff --git a/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts b/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts index fd0c17a3be465..fdbfc1e2e947d 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts +++ b/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts @@ -6,6 +6,7 @@ import { PackageInstallation } from './package-installation'; import { LockFile, PackageManager } from './package-manager'; import { BundlingOptions, OutputFormat, SourceMapMode } from './types'; import { exec, extractDependencies, findUp, getTsconfigCompilerOptions } from './util'; +import { BundlingFileCopyVariant } from '@aws-cdk/core'; const ESBUILD_MAJOR_VERSION = '0'; @@ -43,6 +44,12 @@ export interface BundlingProps extends BundlingOptions { */ readonly preCompilation?: boolean + /** + * Which option to use to copy the source files to the docker container and output files back + * @default - BIND_MOUNT + */ + readonly fileCopyVariant?: BundlingFileCopyVariant; + } /** @@ -83,6 +90,7 @@ export class Bundling implements cdk.BundlingOptions { public readonly securityOpt?: string; public readonly network?: string; public readonly local?: cdk.ILocalBundling; + public readonly fileCopyVariant?: cdk.BundlingFileCopyVariant; private readonly projectRoot: string; private readonly relativeEntryPath: string; @@ -154,6 +162,7 @@ export class Bundling implements cdk.BundlingOptions { this.user = props.user; this.securityOpt = props.securityOpt; this.network = props.network; + this.fileCopyVariant = props.fileCopyVariant; // Local bundling if (!props.forceDockerBundling) { // only if Docker is not forced diff --git a/packages/@aws-cdk/aws-lambda-nodejs/lib/types.ts b/packages/@aws-cdk/aws-lambda-nodejs/lib/types.ts index 0ca72b4149b1f..b1e3e23a907d4 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/lib/types.ts +++ b/packages/@aws-cdk/aws-lambda-nodejs/lib/types.ts @@ -1,4 +1,4 @@ -import { DockerImage, DockerRunOptions } from '@aws-cdk/core'; +import { BundlingFileCopyVariant, DockerImage, DockerRunOptions } from '@aws-cdk/core'; /** * Bundling options @@ -301,6 +301,12 @@ export interface BundlingOptions extends DockerRunOptions { * @default - no code is injected */ readonly inject?: string[] + + /** + * Which option to use to copy the source files to the docker container and output files back + * @default - BIND_MOUNT + */ + readonly fileCopyVariant?: BundlingFileCopyVariant; } /** diff --git a/packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts b/packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts index 9679530a01189..be355ec767c37 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts +++ b/packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts @@ -2,7 +2,7 @@ import * as child_process from 'child_process'; import * as os from 'os'; import * as path from 'path'; import { Architecture, Code, Runtime } from '@aws-cdk/aws-lambda'; -import { AssetHashType, DockerImage } from '@aws-cdk/core'; +import { AssetHashType, BundlingFileCopyVariant, DockerImage } from '@aws-cdk/core'; import { version as delayVersion } from 'delay/package.json'; import { Bundling } from '../lib/bundling'; import { PackageInstallation } from '../lib/package-installation'; @@ -802,3 +802,22 @@ test('Custom bundling network', () => { }), }); }); + +test('Custom bundling file copy variant', () => { + Bundling.bundle({ + entry, + projectRoot, + depsLockFilePath, + runtime: Runtime.NODEJS_14_X, + architecture: Architecture.X86_64, + forceDockerBundling: true, + fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY, + }); + + expect(Code.fromAsset).toHaveBeenCalledWith('/project', { + assetHashType: AssetHashType.OUTPUT, + bundling: expect.objectContaining({ + fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY, + }), + }); +}); From dca7457d75d97872a289f5703a3b00eaefd45045 Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Thu, 5 Jan 2023 16:34:18 +0100 Subject: [PATCH 10/34] fix import order --- packages/@aws-cdk/aws-lambda-go/lib/bundling.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts b/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts index 3caca1a13fc05..104620e33ff6b 100644 --- a/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts +++ b/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts @@ -2,9 +2,9 @@ import * as os from 'os'; import * as path from 'path'; import { Architecture, AssetCode, Code, Runtime } from '@aws-cdk/aws-lambda'; import * as cdk from '@aws-cdk/core'; +import { BundlingFileCopyVariant } from '@aws-cdk/core'; import { BundlingOptions } from './types'; import { exec, findUp, getGoBuildVersion } from './util'; -import { BundlingFileCopyVariant } from '@aws-cdk/core'; /** * Options for bundling From e804023fa6d2df1b3272a638027c02a8e220a563 Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Thu, 5 Jan 2023 16:52:59 +0100 Subject: [PATCH 11/34] add readme --- packages/@aws-cdk/aws-lambda-go/README.md | 14 ++++++++++++++ packages/@aws-cdk/aws-lambda-nodejs/README.md | 13 +++++++++++++ packages/@aws-cdk/aws-lambda-python/README.md | 17 +++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/packages/@aws-cdk/aws-lambda-go/README.md b/packages/@aws-cdk/aws-lambda-go/README.md index 748ab32256ec0..a311e578704cd 100644 --- a/packages/@aws-cdk/aws-lambda-go/README.md +++ b/packages/@aws-cdk/aws-lambda-go/README.md @@ -284,3 +284,17 @@ and Go only includes dependencies that are used in the executable. So in this ca if `cmd/api` used the `auth` & `middleware` packages, but `cmd/anotherApi` did not, then an update to `auth` or `middleware` would only trigger an update to the `cmd/api` Lambda Function. + +## Docker based bundling in complex Docker configurations + +By default the input and output of Docker based bundling is handled via bind mounts. +In situtations where this does not work, like Docker-in-Docker setups or when using a remote Docker socket, you can configure an alternative, but slower, variant that also works in these situations. + + ```ts +new go.GoFunction(this, 'GoFunction', { + entry: 'app/cmd/api', + bundling: { + fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY, + }, +}); +``` diff --git a/packages/@aws-cdk/aws-lambda-nodejs/README.md b/packages/@aws-cdk/aws-lambda-nodejs/README.md index 24d6582d67732..07ce5d26c65a9 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/README.md +++ b/packages/@aws-cdk/aws-lambda-nodejs/README.md @@ -337,3 +337,16 @@ new nodejs.NodejsFunction(this, 'my-handler', { If you chose to customize the hash, you will need to make sure it is updated every time the asset changes, or otherwise it is possible that some deployments will not be invalidated. + +## Docker based bundling in complex Docker configurations + +By default the input and output of Docker based bundling is handled via bind mounts. +In situtations where this does not work, like Docker-in-Docker setups or when using a remote Docker socket, you can configure an alternative, but slower, variant that also works in these situations. + + ```ts + new nodejs.NodejsFunction(this, 'my-handler', { + bundling: { + fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY, + }, +}); +``` diff --git a/packages/@aws-cdk/aws-lambda-python/README.md b/packages/@aws-cdk/aws-lambda-python/README.md index f7ca29de39e0a..27e7d384a14a2 100644 --- a/packages/@aws-cdk/aws-lambda-python/README.md +++ b/packages/@aws-cdk/aws-lambda-python/README.md @@ -252,3 +252,20 @@ an array of commands to run. Commands are chained with `&&`. The commands will run in the environment in which bundling occurs: inside the container for Docker bundling or on the host OS for local bundling. + +## Docker based bundling in complex Docker configurations + +By default the input and output of Docker based bundling is handled via bind mounts. +In situtations where this does not work, like Docker-in-Docker setups or when using a remote Docker socket, you can configure an alternative, but slower, variant that also works in these situations. + + ```ts + const entry = '/path/to/function'; + +new python.PythonFunction(this, 'function', { + entry, + runtime: Runtime.PYTHON_3_8, + bundling: { + fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY, + }, +}); +``` From e66743622707ced510dbeda9840db13e3171354b Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Mon, 9 Jan 2023 08:24:28 +0100 Subject: [PATCH 12/34] add import to readme --- packages/@aws-cdk/aws-lambda-go/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/@aws-cdk/aws-lambda-go/README.md b/packages/@aws-cdk/aws-lambda-go/README.md index a311e578704cd..8f1f15c7fc2a5 100644 --- a/packages/@aws-cdk/aws-lambda-go/README.md +++ b/packages/@aws-cdk/aws-lambda-go/README.md @@ -291,6 +291,7 @@ By default the input and output of Docker based bundling is handled via bind mou In situtations where this does not work, like Docker-in-Docker setups or when using a remote Docker socket, you can configure an alternative, but slower, variant that also works in these situations. ```ts +import { BundlingFileCopyVariant } from '@aws-cdk/core'; new go.GoFunction(this, 'GoFunction', { entry: 'app/cmd/api', bundling: { From 13cba3282028cc77bd864fa906998036b6b3b369 Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Fri, 20 Jan 2023 09:09:24 +0100 Subject: [PATCH 13/34] Update packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts Co-authored-by: Momo Kornher --- packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts b/packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts index 6b6ebb0439a64..f24462234a735 100644 --- a/packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts +++ b/packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts @@ -478,4 +478,4 @@ test('Custom bundling file copy variant', () => { fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY, }), }); -}); \ No newline at end of file +}); From 1ca84966c760376a7b07fe4bbc17295b70f823ab Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Fri, 20 Jan 2023 09:09:35 +0100 Subject: [PATCH 14/34] Update packages/@aws-cdk/core/lib/bundling.ts Co-authored-by: Momo Kornher --- packages/@aws-cdk/core/lib/bundling.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/core/lib/bundling.ts b/packages/@aws-cdk/core/lib/bundling.ts index 3a12906ccb0fd..fb3a940a552ec 100644 --- a/packages/@aws-cdk/core/lib/bundling.ts +++ b/packages/@aws-cdk/core/lib/bundling.ts @@ -172,7 +172,7 @@ export enum BundlingFileCopyVariant { * The source and output folders will be mounted as bind mount from the host system * This is faster and simpler, but less portable than the other option. */ - BIND_MOUNT = 'bind-mount', + BIND_MOUNT = 'BIND_MOUNT', } From 813c8b77b22f60d58c4dfb0b16bd46664dc1a4bc Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Fri, 20 Jan 2023 09:09:50 +0100 Subject: [PATCH 15/34] Update packages/@aws-cdk/core/lib/bundling.ts Co-authored-by: Momo Kornher --- packages/@aws-cdk/core/lib/bundling.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/core/lib/bundling.ts b/packages/@aws-cdk/core/lib/bundling.ts index fb3a940a552ec..c0eb35ecf5aa1 100644 --- a/packages/@aws-cdk/core/lib/bundling.ts +++ b/packages/@aws-cdk/core/lib/bundling.ts @@ -166,7 +166,7 @@ export enum BundlingFileCopyVariant { * Creates temporary volumes and docker containers * This is slower, but works also in more complex situations with remote or shared docker sockets. */ - DOCKER_COPY = 'docker-copy', + VOLUME_COPY = 'VOLUME_COPY', /** * The source and output folders will be mounted as bind mount from the host system From 871da219a93756bf2ee0decd84de9121dfebae8e Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Fri, 20 Jan 2023 09:10:18 +0100 Subject: [PATCH 16/34] Update packages/@aws-cdk/core/lib/bundling.ts Co-authored-by: Momo Kornher --- packages/@aws-cdk/core/lib/bundling.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/core/lib/bundling.ts b/packages/@aws-cdk/core/lib/bundling.ts index c0eb35ecf5aa1..b4af34236f629 100644 --- a/packages/@aws-cdk/core/lib/bundling.ts +++ b/packages/@aws-cdk/core/lib/bundling.ts @@ -111,8 +111,8 @@ export interface BundlingOptions { readonly network?: string; /** - * Which option to use to copy the source files to the docker container and output files back - * @default - BIND_MOUNT + * The Which option to use to copy the source files to the docker container and output files back + * @default - BundlingFileCopyVariant.BIND_MOUNT */ readonly fileCopyVariant?: BundlingFileCopyVariant; } From ed23041ce214c2981c8919155d7cf796fafd4ed8 Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Fri, 20 Jan 2023 09:10:58 +0100 Subject: [PATCH 17/34] Update packages/@aws-cdk/core/lib/bundling.ts Co-authored-by: Momo Kornher --- packages/@aws-cdk/core/lib/bundling.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/core/lib/bundling.ts b/packages/@aws-cdk/core/lib/bundling.ts index b4af34236f629..65466634dd2be 100644 --- a/packages/@aws-cdk/core/lib/bundling.ts +++ b/packages/@aws-cdk/core/lib/bundling.ts @@ -562,7 +562,7 @@ export interface DockerBuildOptions { /** * Provides a helper container for copying bundling related files to specific input and output volumes */ -export class DockerImageBundlingCopyHelper { +export class AssetStagingVolumeCopy { /** * Name of the Docker volume that is used for the asset input */ From 6bc93e1c0e137527aa323f9bc515046bc5d32f24 Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Fri, 20 Jan 2023 09:29:26 +0100 Subject: [PATCH 18/34] address first MR comments --- packages/@aws-cdk/aws-lambda-go/lib/bundling.ts | 2 +- packages/@aws-cdk/aws-lambda-go/lib/types.ts | 2 +- .../@aws-cdk/aws-lambda-go/test/bundling.test.ts | 4 ++-- packages/@aws-cdk/aws-lambda-nodejs/README.md | 1 + .../@aws-cdk/aws-lambda-nodejs/lib/bundling.ts | 2 +- packages/@aws-cdk/aws-lambda-nodejs/lib/types.ts | 2 +- .../aws-lambda-nodejs/test/bundling.test.ts | 4 ++-- packages/@aws-cdk/aws-lambda-python/README.md | 1 + .../@aws-cdk/aws-lambda-python/lib/bundling.ts | 2 +- packages/@aws-cdk/aws-lambda-python/lib/types.ts | 2 +- .../aws-lambda-python/test/bundling.test.ts | 4 ++-- .../test/integ.function.dockercopy.ts | 2 +- packages/@aws-cdk/core/lib/asset-staging.ts | 8 ++++---- packages/@aws-cdk/core/test/bundling.test.ts | 14 +++++++------- packages/@aws-cdk/core/test/staging.test.ts | 8 ++++---- 15 files changed, 30 insertions(+), 28 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts b/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts index 104620e33ff6b..71c8e3ab554b5 100644 --- a/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts +++ b/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts @@ -64,7 +64,7 @@ export interface BundlingProps extends BundlingOptions { /** * Which option to use to copy the source files to the docker container and output files back - * @default - BIND_MOUNT + * @default - BundlingFileCopyVariant.BIND_MOUNT */ readonly fileCopyVariant?: BundlingFileCopyVariant; } diff --git a/packages/@aws-cdk/aws-lambda-go/lib/types.ts b/packages/@aws-cdk/aws-lambda-go/lib/types.ts index ec53819c7c38b..f7ac7bef9155f 100644 --- a/packages/@aws-cdk/aws-lambda-go/lib/types.ts +++ b/packages/@aws-cdk/aws-lambda-go/lib/types.ts @@ -114,7 +114,7 @@ export interface BundlingOptions extends DockerRunOptions { /** * Which option to use to copy the source files to the docker container and output files back - * @default - BIND_MOUNT + * @default - BundlingFileCopyVariant.BIND_MOUNT */ readonly fileCopyVariant?: BundlingFileCopyVariant; } diff --git a/packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts b/packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts index f24462234a735..0494c55bfe885 100644 --- a/packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts +++ b/packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts @@ -469,13 +469,13 @@ test('Custom bundling file copy variant', () => { runtime: Runtime.GO_1_X, architecture: Architecture.X86_64, forcedDockerBundling: true, - fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY, + fileCopyVariant: BundlingFileCopyVariant.VOLUME_COPY, }); expect(Code.fromAsset).toHaveBeenCalledWith('/project', { assetHashType: AssetHashType.OUTPUT, bundling: expect.objectContaining({ - fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY, + fileCopyVariant: BundlingFileCopyVariant.VOLUME_COPY, }), }); }); diff --git a/packages/@aws-cdk/aws-lambda-nodejs/README.md b/packages/@aws-cdk/aws-lambda-nodejs/README.md index 07ce5d26c65a9..396d3c4762d59 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/README.md +++ b/packages/@aws-cdk/aws-lambda-nodejs/README.md @@ -344,6 +344,7 @@ By default the input and output of Docker based bundling is handled via bind mou In situtations where this does not work, like Docker-in-Docker setups or when using a remote Docker socket, you can configure an alternative, but slower, variant that also works in these situations. ```ts +import { BundlingFileCopyVariant } from '@aws-cdk/core'; new nodejs.NodejsFunction(this, 'my-handler', { bundling: { fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY, diff --git a/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts b/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts index fdbfc1e2e947d..68e6d153bc59e 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts +++ b/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts @@ -46,7 +46,7 @@ export interface BundlingProps extends BundlingOptions { /** * Which option to use to copy the source files to the docker container and output files back - * @default - BIND_MOUNT + * @default - BundlingFileCopyVariant.BIND_MOUNT */ readonly fileCopyVariant?: BundlingFileCopyVariant; diff --git a/packages/@aws-cdk/aws-lambda-nodejs/lib/types.ts b/packages/@aws-cdk/aws-lambda-nodejs/lib/types.ts index b1e3e23a907d4..0b5bce1737c7c 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/lib/types.ts +++ b/packages/@aws-cdk/aws-lambda-nodejs/lib/types.ts @@ -304,7 +304,7 @@ export interface BundlingOptions extends DockerRunOptions { /** * Which option to use to copy the source files to the docker container and output files back - * @default - BIND_MOUNT + * @default - BundlingFileCopyVariant.BIND_MOUNT */ readonly fileCopyVariant?: BundlingFileCopyVariant; } diff --git a/packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts b/packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts index be355ec767c37..f2332b29b3d01 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts +++ b/packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts @@ -811,13 +811,13 @@ test('Custom bundling file copy variant', () => { runtime: Runtime.NODEJS_14_X, architecture: Architecture.X86_64, forceDockerBundling: true, - fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY, + fileCopyVariant: BundlingFileCopyVariant.VOLUME_COPY, }); expect(Code.fromAsset).toHaveBeenCalledWith('/project', { assetHashType: AssetHashType.OUTPUT, bundling: expect.objectContaining({ - fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY, + fileCopyVariant: BundlingFileCopyVariant.VOLUME_COPY, }), }); }); diff --git a/packages/@aws-cdk/aws-lambda-python/README.md b/packages/@aws-cdk/aws-lambda-python/README.md index 27e7d384a14a2..df6ababc1fb02 100644 --- a/packages/@aws-cdk/aws-lambda-python/README.md +++ b/packages/@aws-cdk/aws-lambda-python/README.md @@ -259,6 +259,7 @@ By default the input and output of Docker based bundling is handled via bind mou In situtations where this does not work, like Docker-in-Docker setups or when using a remote Docker socket, you can configure an alternative, but slower, variant that also works in these situations. ```ts + import { BundlingFileCopyVariant } from '@aws-cdk/core'; const entry = '/path/to/function'; new python.PythonFunction(this, 'function', { diff --git a/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts b/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts index e7a629907ecd1..2675e33a61150 100644 --- a/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts +++ b/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts @@ -44,7 +44,7 @@ export interface BundlingProps extends BundlingOptions { /** * Which option to use to copy the source files to the docker container and output files back - * @default - BIND_MOUNT + * @default - BundlingFileCopyVariant.BIND_MOUNT */ fileCopyVariant?: BundlingFileCopyVariant } diff --git a/packages/@aws-cdk/aws-lambda-python/lib/types.ts b/packages/@aws-cdk/aws-lambda-python/lib/types.ts index 625a232d82396..1f384fa6c8de0 100644 --- a/packages/@aws-cdk/aws-lambda-python/lib/types.ts +++ b/packages/@aws-cdk/aws-lambda-python/lib/types.ts @@ -89,7 +89,7 @@ export interface BundlingOptions extends DockerRunOptions { /** * Which option to use to copy the source files to the docker container and output files back - * @default - BIND_MOUNT + * @default - BundlingFileCopyVariant.BIND_MOUNT */ readonly fileCopyVariant?: BundlingFileCopyVariant; } diff --git a/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts b/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts index f3b40596d47a7..ef7bd5b9950d7 100644 --- a/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts +++ b/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts @@ -379,13 +379,13 @@ test('Bundling with docker copy variant', () => { Bundling.bundle({ entry: entry, runtime: Runtime.PYTHON_3_7, - fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY, + fileCopyVariant: BundlingFileCopyVariant.VOLUME_COPY, }); expect(Code.fromAsset).toHaveBeenCalledWith(entry, expect.objectContaining({ bundling: expect.objectContaining({ - fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY, + fileCopyVariant: BundlingFileCopyVariant.VOLUME_COPY, }), })); }); diff --git a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.ts b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.ts index 200f3ffa33a1b..d66263e72913d 100644 --- a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.ts +++ b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.ts @@ -22,7 +22,7 @@ class TestStack extends Stack { entry: path.join(__dirname, 'lambda-handler-dockercopy'), runtime: Runtime.PYTHON_3_9, bundling: { - fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY, + fileCopyVariant: BundlingFileCopyVariant.VOLUME_COPY, }, }); this.functionName = fn.functionName; diff --git a/packages/@aws-cdk/core/lib/asset-staging.ts b/packages/@aws-cdk/core/lib/asset-staging.ts index de36452334948..f96c191a2262a 100644 --- a/packages/@aws-cdk/core/lib/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/asset-staging.ts @@ -5,7 +5,7 @@ import * as cxapi from '@aws-cdk/cx-api'; import { Construct } from 'constructs'; import * as fs from 'fs-extra'; import { AssetHashType, AssetOptions, FileAssetPackaging } from './assets'; -import { BundlingFileCopyVariant, BundlingOptions, BundlingOutput, DockerImageBundlingCopyHelper, DockerVolume } from './bundling'; +import { BundlingFileCopyVariant, BundlingOptions, BundlingOutput, AssetStagingVolumeCopy, DockerVolume } from './bundling'; import { FileSystem, FingerprintOptions } from './fs'; import { clearLargeFileFingerprintCache } from './fs/fingerprint'; import { Names } from './names'; @@ -451,8 +451,8 @@ export class AssetStaging extends Construct { let volumes: DockerVolume[] = []; let volumesFrom: string[] = options.volumesFrom ?? []; - const helperContainer = new DockerImageBundlingCopyHelper(); - if (options.fileCopyVariant?.valueOf() === BundlingFileCopyVariant.DOCKER_COPY.valueOf()) { + const helperContainer = new AssetStagingVolumeCopy(); + if (options.fileCopyVariant?.valueOf() === BundlingFileCopyVariant.VOLUME_COPY.valueOf()) { volumes = options.volumes ?? []; volumesFrom = [helperContainer.copyContainerName, ...options.volumesFrom ?? []]; helperContainer.prepareVolumes(); @@ -483,7 +483,7 @@ export class AssetStaging extends Construct { volumesFrom, }); - if (options.fileCopyVariant?.valueOf() === BundlingFileCopyVariant.DOCKER_COPY.valueOf()) { + if (options.fileCopyVariant?.valueOf() === BundlingFileCopyVariant.VOLUME_COPY.valueOf()) { helperContainer.copyOutputTo(bundleDir); helperContainer.cleanHelperContainer(); helperContainer.cleanVolumes(); diff --git a/packages/@aws-cdk/core/test/bundling.test.ts b/packages/@aws-cdk/core/test/bundling.test.ts index 1fc20efcdaf31..503c886c125d3 100644 --- a/packages/@aws-cdk/core/test/bundling.test.ts +++ b/packages/@aws-cdk/core/test/bundling.test.ts @@ -2,7 +2,7 @@ import * as child_process from 'child_process'; import * as crypto from 'crypto'; import * as path from 'path'; import * as sinon from 'sinon'; -import { AssetStaging, DockerImage, DockerImageBundlingCopyHelper, FileSystem } from '../lib'; +import { AssetStaging, DockerImage, AssetStagingVolumeCopy, FileSystem } from '../lib'; describe('bundling', () => { afterEach(() => { @@ -610,7 +610,7 @@ describe('bundling', () => { output: ['stdout', 'stderr'], signal: null, }); - const helper = new DockerImageBundlingCopyHelper(); + const helper = new AssetStagingVolumeCopy(); helper.prepareVolumes(); // THEN @@ -635,7 +635,7 @@ describe('bundling', () => { output: ['stdout', 'stderr'], signal: null, }); - const helper = new DockerImageBundlingCopyHelper(); + const helper = new AssetStagingVolumeCopy(); helper.cleanVolumes(); // THEN @@ -660,7 +660,7 @@ describe('bundling', () => { output: ['stdout', 'stderr'], signal: null, }); - const helper = new DockerImageBundlingCopyHelper(); + const helper = new AssetStagingVolumeCopy(); const user = '1000'; helper.startHelperContainer(user); @@ -688,7 +688,7 @@ describe('bundling', () => { output: ['stdout', 'stderr'], signal: null, }); - const helper = new DockerImageBundlingCopyHelper(); + const helper = new AssetStagingVolumeCopy(); helper.cleanHelperContainer(); // THEN @@ -708,7 +708,7 @@ describe('bundling', () => { output: ['stdout', 'stderr'], signal: null, }); - const helper = new DockerImageBundlingCopyHelper(); + const helper = new AssetStagingVolumeCopy(); const dir = '/test/dir'; helper.copyInputFrom(dir); @@ -729,7 +729,7 @@ describe('bundling', () => { output: ['stdout', 'stderr'], signal: null, }); - const helper = new DockerImageBundlingCopyHelper(); + const helper = new AssetStagingVolumeCopy(); const dir = '/test/dir'; helper.copyOutputTo(dir); diff --git a/packages/@aws-cdk/core/test/staging.test.ts b/packages/@aws-cdk/core/test/staging.test.ts index 001bcd13a2eee..9c9c0f02d53d3 100644 --- a/packages/@aws-cdk/core/test/staging.test.ts +++ b/packages/@aws-cdk/core/test/staging.test.ts @@ -1293,21 +1293,21 @@ describe('staging with docker cp', () => { bundling: { image: DockerImage.fromRegistry('alpine'), command: [DockerStubCommand.VOLUME_SINGLE_ARCHIVE], - fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY, + fileCopyVariant: BundlingFileCopyVariant.VOLUME_COPY, }, }); // THEN const assembly = app.synth(); expect(fs.readdirSync(assembly.directory)).toEqual([ - 'asset.358c99dcb340194559aa346ecba33e7f22d3e9607bb2885623325f5eb6e0389f', // this is the bundle dir - 'asset.358c99dcb340194559aa346ecba33e7f22d3e9607bb2885623325f5eb6e0389f.zip', + 'asset.115de8f71f52dcd3958586818e7e8e42be04db3039bc14c0501e0d26efbd653a', // this is the bundle dir + 'asset.115de8f71f52dcd3958586818e7e8e42be04db3039bc14c0501e0d26efbd653a.zip', 'cdk.out', 'manifest.json', 'stack.template.json', 'tree.json', ]); - expect(fs.readdirSync(path.join(assembly.directory, 'asset.358c99dcb340194559aa346ecba33e7f22d3e9607bb2885623325f5eb6e0389f'))).toEqual([ + expect(fs.readdirSync(path.join(assembly.directory, 'asset.115de8f71f52dcd3958586818e7e8e42be04db3039bc14c0501e0d26efbd653a'))).toEqual([ 'test.zip', // bundle dir with "touched" bundled output file ]); expect(staging.packaging).toEqual(FileAssetPackaging.FILE); From 7b3798e0b0da1372d51bb669c6e1adf49ec61bae Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Fri, 20 Jan 2023 10:17:22 +0100 Subject: [PATCH 19/34] fix readme --- packages/@aws-cdk/aws-lambda-go/README.md | 2 +- packages/@aws-cdk/aws-lambda-nodejs/README.md | 2 +- packages/@aws-cdk/aws-lambda-python/README.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-go/README.md b/packages/@aws-cdk/aws-lambda-go/README.md index 8f1f15c7fc2a5..84561b50c52f9 100644 --- a/packages/@aws-cdk/aws-lambda-go/README.md +++ b/packages/@aws-cdk/aws-lambda-go/README.md @@ -295,7 +295,7 @@ import { BundlingFileCopyVariant } from '@aws-cdk/core'; new go.GoFunction(this, 'GoFunction', { entry: 'app/cmd/api', bundling: { - fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY, + fileCopyVariant: BundlingFileCopyVariant.VOLUME_COPY, }, }); ``` diff --git a/packages/@aws-cdk/aws-lambda-nodejs/README.md b/packages/@aws-cdk/aws-lambda-nodejs/README.md index 396d3c4762d59..825e57cc367a5 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/README.md +++ b/packages/@aws-cdk/aws-lambda-nodejs/README.md @@ -347,7 +347,7 @@ In situtations where this does not work, like Docker-in-Docker setups or when us import { BundlingFileCopyVariant } from '@aws-cdk/core'; new nodejs.NodejsFunction(this, 'my-handler', { bundling: { - fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY, + fileCopyVariant: BundlingFileCopyVariant.VOLUME_COPY, }, }); ``` diff --git a/packages/@aws-cdk/aws-lambda-python/README.md b/packages/@aws-cdk/aws-lambda-python/README.md index df6ababc1fb02..abcd25aaad860 100644 --- a/packages/@aws-cdk/aws-lambda-python/README.md +++ b/packages/@aws-cdk/aws-lambda-python/README.md @@ -266,7 +266,7 @@ new python.PythonFunction(this, 'function', { entry, runtime: Runtime.PYTHON_3_8, bundling: { - fileCopyVariant: BundlingFileCopyVariant.DOCKER_COPY, + fileCopyVariant: BundlingFileCopyVariant.VOLUME_COPY, }, }); ``` From 54b3a80e4d144ecf1974a1d207362bfd1ca914f3 Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Fri, 20 Jan 2023 11:53:22 +0100 Subject: [PATCH 20/34] WIP: restructure --- .../@aws-cdk/aws-lambda-go/lib/bundling.ts | 6 +- .../aws-lambda-nodejs/lib/bundling.ts | 4 +- .../aws-lambda-python/lib/bundling.ts | 4 +- packages/@aws-cdk/core/lib/asset-staging.ts | 54 ++++--------- packages/@aws-cdk/core/lib/bundling.ts | 80 ++++++++++++++++++- packages/@aws-cdk/core/test/staging.test.ts | 2 +- 6 files changed, 101 insertions(+), 49 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts b/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts index 71c8e3ab554b5..abdb021d45db1 100644 --- a/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts +++ b/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts @@ -92,7 +92,7 @@ export class Bundling implements cdk.BundlingOptions { user: bundling.user, securityOpt: bundling.securityOpt, network: bundling.network, - fileCopyVariant: bundling.fileCopyVariant, + assetStagingType: bundling.assetStagingType, }, }); } @@ -115,7 +115,7 @@ export class Bundling implements cdk.BundlingOptions { public readonly user?: string; public readonly securityOpt?: string; public readonly network?: string; - public readonly fileCopyVariant?: cdk.BundlingFileCopyVariant; + public readonly assetStagingType?: cdk.BundlingFileCopyVariant; private readonly relativeEntryPath: string; @@ -163,7 +163,7 @@ export class Bundling implements cdk.BundlingOptions { this.user = props.user; this.securityOpt = props.securityOpt; this.network = props.network; - this.fileCopyVariant = props.fileCopyVariant; + this.assetStagingType = props.fileCopyVariant; // Local bundling if (!props.forcedDockerBundling) { // only if Docker is not forced diff --git a/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts b/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts index 68e6d153bc59e..b1d2e1bb347cd 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts +++ b/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts @@ -90,7 +90,7 @@ export class Bundling implements cdk.BundlingOptions { public readonly securityOpt?: string; public readonly network?: string; public readonly local?: cdk.ILocalBundling; - public readonly fileCopyVariant?: cdk.BundlingFileCopyVariant; + public readonly assetStagingType?: cdk.BundlingFileCopyVariant; private readonly projectRoot: string; private readonly relativeEntryPath: string; @@ -162,7 +162,7 @@ export class Bundling implements cdk.BundlingOptions { this.user = props.user; this.securityOpt = props.securityOpt; this.network = props.network; - this.fileCopyVariant = props.fileCopyVariant; + this.assetStagingType = props.fileCopyVariant; // Local bundling if (!props.forceDockerBundling) { // only if Docker is not forced diff --git a/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts b/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts index 2675e33a61150..3b141c1536577 100644 --- a/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts +++ b/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts @@ -72,7 +72,7 @@ export class Bundling implements CdkBundlingOptions { public readonly user?: string; public readonly securityOpt?: string; public readonly network?: string; - public readonly fileCopyVariant?: BundlingFileCopyVariant; + public readonly assetStagingType?: BundlingFileCopyVariant; constructor(props: BundlingProps) { const { @@ -111,7 +111,7 @@ export class Bundling implements CdkBundlingOptions { this.user = props.user; this.securityOpt = props.securityOpt; this.network = props.network; - this.fileCopyVariant = props.fileCopyVariant; + this.assetStagingType = props.fileCopyVariant; } private createBundlingCommand(options: BundlingCommandOptions): string[] { diff --git a/packages/@aws-cdk/core/lib/asset-staging.ts b/packages/@aws-cdk/core/lib/asset-staging.ts index f96c191a2262a..c6015baca0d35 100644 --- a/packages/@aws-cdk/core/lib/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/asset-staging.ts @@ -5,7 +5,7 @@ import * as cxapi from '@aws-cdk/cx-api'; import { Construct } from 'constructs'; import * as fs from 'fs-extra'; import { AssetHashType, AssetOptions, FileAssetPackaging } from './assets'; -import { BundlingFileCopyVariant, BundlingOptions, BundlingOutput, AssetStagingVolumeCopy, DockerVolume } from './bundling'; +import { BundlingFileCopyVariant, BundlingOptions, BundlingOutput, AssetStagingVolumeCopy, AssetStagingBindMount } from './bundling'; import { FileSystem, FingerprintOptions } from './fs'; import { clearLargeFileFingerprintCache } from './fs/fingerprint'; import { Names } from './names'; @@ -448,45 +448,21 @@ export class AssetStaging extends Construct { : '1000:1000'; } - let volumes: DockerVolume[] = []; - let volumesFrom: string[] = options.volumesFrom ?? []; - - const helperContainer = new AssetStagingVolumeCopy(); - if (options.fileCopyVariant?.valueOf() === BundlingFileCopyVariant.VOLUME_COPY.valueOf()) { - volumes = options.volumes ?? []; - volumesFrom = [helperContainer.copyContainerName, ...options.volumesFrom ?? []]; - helperContainer.prepareVolumes(); - helperContainer.startHelperContainer(user); - helperContainer.copyInputFrom(this.sourcePath); - } else { - volumes = [ - { - hostPath: this.sourcePath, - containerPath: AssetStaging.BUNDLING_INPUT_DIR, - }, - { - hostPath: bundleDir, - containerPath: AssetStaging.BUNDLING_OUTPUT_DIR, - }, - ...options.volumes ?? [], - ]; - } - - options.image.run({ - command: options.command, + const assetStagingOptions = { user, - environment: options.environment, - entrypoint: options.entrypoint, - workingDirectory: options.workingDirectory ?? AssetStaging.BUNDLING_INPUT_DIR, - securityOpt: options.securityOpt ?? '', - volumes, - volumesFrom, - }); - - if (options.fileCopyVariant?.valueOf() === BundlingFileCopyVariant.VOLUME_COPY.valueOf()) { - helperContainer.copyOutputTo(bundleDir); - helperContainer.cleanHelperContainer(); - helperContainer.cleanVolumes(); + sourcePath: this.sourcePath, + bundleDir, + ...options, + }; + + switch (options.assetStagingType) { + case BundlingFileCopyVariant.VOLUME_COPY: + new AssetStagingVolumeCopy(assetStagingOptions).run(); + break; + case BundlingFileCopyVariant.BIND_MOUNT: + default: + new AssetStagingBindMount(assetStagingOptions).run(); + break; } } } catch (err) { diff --git a/packages/@aws-cdk/core/lib/bundling.ts b/packages/@aws-cdk/core/lib/bundling.ts index 65466634dd2be..d82191fad0301 100644 --- a/packages/@aws-cdk/core/lib/bundling.ts +++ b/packages/@aws-cdk/core/lib/bundling.ts @@ -114,7 +114,7 @@ export interface BundlingOptions { * The Which option to use to copy the source files to the docker container and output files back * @default - BundlingFileCopyVariant.BIND_MOUNT */ - readonly fileCopyVariant?: BundlingFileCopyVariant; + readonly assetStagingType?: BundlingFileCopyVariant; } /** @@ -559,6 +559,55 @@ export interface DockerBuildOptions { readonly targetStage?: string; } +/** + * Options for Docker based bundling of assets + */ +export interface AssetStagingCopyOptions extends BundlingOptions { + /** + * Path where the source files are located + */ + readonly sourcePath: string; + /** + * Path where the output files should be stored + */ + readonly bundleDir: string; +} + +/** + * Bundles files with bind mount as copy method + */ +export class AssetStagingBindMount { + private options: AssetStagingCopyOptions + constructor(options: AssetStagingCopyOptions) { + this.options = options; + } + /** + * Bundle files with bind mount as copy method + */ + public run() { + this.options.image.run({ + command: this.options.command, + user: this.options.user, + environment: this.options.environment, + entrypoint: this.options.entrypoint, + workingDirectory: this.options.workingDirectory ?? AssetStaging.BUNDLING_INPUT_DIR, + securityOpt: this.options.securityOpt ?? '', + volumesFrom: this.options.volumesFrom, + volumes: [ + { + hostPath: this.options.sourcePath, + containerPath: AssetStaging.BUNDLING_INPUT_DIR, + }, + { + hostPath: this.options.bundleDir, + containerPath: AssetStaging.BUNDLING_OUTPUT_DIR, + }, + ...this.options.volumes ?? [], + ], + }); + } +} + /** * Provides a helper container for copying bundling related files to specific input and output volumes */ @@ -571,16 +620,18 @@ export class AssetStagingVolumeCopy { * Name of the Docker volume that is used for the asset output */ private outputVolumeName: string; + private options: AssetStagingCopyOptions /** * Name of the Docker helper container to copy files into the volume */ public copyContainerName: string; - constructor() { + constructor(options: AssetStagingCopyOptions) { const copySuffix = crypto.randomBytes(12).toString('hex'); this.inputVolumeName = `assetInput${copySuffix}`; this.outputVolumeName = `assetOutput${copySuffix}`; this.copyContainerName = `copyContainer${copySuffix}`; + this.options = options; } /** @@ -639,6 +690,31 @@ export class AssetStagingVolumeCopy { public copyOutputTo(outputPath: string) { dockerExec(['cp', `${this.copyContainerName}:${AssetStaging.BUNDLING_OUTPUT_DIR}/.`, outputPath]); } + + /** + * Bundle files with VOLUME_COPY method + */ + public run() { + this.prepareVolumes(); + this.startHelperContainer(this.options.user ?? '1000:1000'); // TODO handle user properly + this.copyInputFrom(this.options.sourcePath); + + this.options.image.run({ + command: this.options.command, + user: this.options.user, + environment: this.options.environment, + entrypoint: this.options.entrypoint, + workingDirectory: this.options.workingDirectory ?? AssetStaging.BUNDLING_INPUT_DIR, + securityOpt: this.options.securityOpt ?? '', + volumes: this.options.volumes, + volumesFrom: [this.copyContainerName, ...this.options.volumesFrom ?? []], + }); + + this.copyOutputTo(this.options.bundleDir); + this.cleanHelperContainer(); + this.cleanVolumes(); + } + } function flatten(x: string[][]) { diff --git a/packages/@aws-cdk/core/test/staging.test.ts b/packages/@aws-cdk/core/test/staging.test.ts index 9c9c0f02d53d3..6491eeb0fa4c9 100644 --- a/packages/@aws-cdk/core/test/staging.test.ts +++ b/packages/@aws-cdk/core/test/staging.test.ts @@ -1293,7 +1293,7 @@ describe('staging with docker cp', () => { bundling: { image: DockerImage.fromRegistry('alpine'), command: [DockerStubCommand.VOLUME_SINGLE_ARCHIVE], - fileCopyVariant: BundlingFileCopyVariant.VOLUME_COPY, + assetStagingType: BundlingFileCopyVariant.VOLUME_COPY, }, }); From 5c98592a6c96eaf984fd600d9b3c0df18256cafe Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Mon, 23 Jan 2023 08:49:23 +0100 Subject: [PATCH 21/34] WIP separate bundling --- packages/@aws-cdk/core/lib/asset-staging.ts | 12 --- packages/@aws-cdk/core/lib/bundling.ts | 53 ++++++--- packages/@aws-cdk/core/test/bundling.test.ts | 108 +++---------------- 3 files changed, 52 insertions(+), 121 deletions(-) diff --git a/packages/@aws-cdk/core/lib/asset-staging.ts b/packages/@aws-cdk/core/lib/asset-staging.ts index c6015baca0d35..a9e71e7360d91 100644 --- a/packages/@aws-cdk/core/lib/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/asset-staging.ts @@ -1,5 +1,4 @@ import * as crypto from 'crypto'; -import * as os from 'os'; import * as path from 'path'; import * as cxapi from '@aws-cdk/cx-api'; import { Construct } from 'constructs'; @@ -438,18 +437,7 @@ export class AssetStaging extends Construct { localBundling = options.local?.tryBundle(bundleDir, options); if (!localBundling) { - let user: string; - if (options.user) { - user = options.user; - } else { // Default to current user - const userInfo = os.userInfo(); - user = userInfo.uid !== -1 // uid is -1 on Windows - ? `${userInfo.uid}:${userInfo.gid}` - : '1000:1000'; - } - const assetStagingOptions = { - user, sourcePath: this.sourcePath, bundleDir, ...options, diff --git a/packages/@aws-cdk/core/lib/bundling.ts b/packages/@aws-cdk/core/lib/bundling.ts index d82191fad0301..e516a70d38ac8 100644 --- a/packages/@aws-cdk/core/lib/bundling.ts +++ b/packages/@aws-cdk/core/lib/bundling.ts @@ -1,5 +1,6 @@ import { spawnSync, SpawnSyncOptions } from 'child_process'; import * as crypto from 'crypto'; +import * as os from 'os'; import { isAbsolute, join } from 'path'; import { AssetStaging } from './asset-staging'; import { FileSystem } from './fs'; @@ -573,21 +574,39 @@ export interface AssetStagingCopyOptions extends BundlingOptions { readonly bundleDir: string; } -/** - * Bundles files with bind mount as copy method - */ -export class AssetStagingBindMount { - private options: AssetStagingCopyOptions +class AssetStagingBase { + protected options: AssetStagingCopyOptions constructor(options: AssetStagingCopyOptions) { this.options = options; } + /** + * Determines a useful default user if not given otherwise + */ + protected getUser() { + let user: string; + if (this.options.user) { + user = this.options.user; + } else { // Default to current user + const userInfo = os.userInfo(); + user = userInfo.uid !== -1 // uid is -1 on Windows + ? `${userInfo.uid}:${userInfo.gid}` + : '1000:1000'; + } + return user; + } +} + +/** + * Bundles files with bind mount as copy method + */ +export class AssetStagingBindMount extends AssetStagingBase { /** * Bundle files with bind mount as copy method */ public run() { this.options.image.run({ command: this.options.command, - user: this.options.user, + user: this.getUser(), environment: this.options.environment, entrypoint: this.options.entrypoint, workingDirectory: this.options.workingDirectory ?? AssetStaging.BUNDLING_INPUT_DIR, @@ -611,7 +630,7 @@ export class AssetStagingBindMount { /** * Provides a helper container for copying bundling related files to specific input and output volumes */ -export class AssetStagingVolumeCopy { +export class AssetStagingVolumeCopy extends AssetStagingBase { /** * Name of the Docker volume that is used for the asset input */ @@ -620,24 +639,23 @@ export class AssetStagingVolumeCopy { * Name of the Docker volume that is used for the asset output */ private outputVolumeName: string; - private options: AssetStagingCopyOptions /** * Name of the Docker helper container to copy files into the volume */ public copyContainerName: string; constructor(options: AssetStagingCopyOptions) { + super(options); const copySuffix = crypto.randomBytes(12).toString('hex'); this.inputVolumeName = `assetInput${copySuffix}`; this.outputVolumeName = `assetOutput${copySuffix}`; this.copyContainerName = `copyContainer${copySuffix}`; - this.options = options; } /** * Creates volumes for asset input and output */ - public prepareVolumes() { + private prepareVolumes() { dockerExec(['volume', 'create', this.inputVolumeName]); dockerExec(['volume', 'create', this.outputVolumeName]); } @@ -645,7 +663,7 @@ export class AssetStagingVolumeCopy { /** * Removes volumes for asset input and output */ - public cleanVolumes() { + private cleanVolumes() { dockerExec(['volume', 'rm', this.inputVolumeName]); dockerExec(['volume', 'rm', this.outputVolumeName]); } @@ -654,7 +672,7 @@ export class AssetStagingVolumeCopy { * runs a helper container that holds volumes and does some preparation tasks * @param user The user that will later access these files and needs permissions to do so */ - public startHelperContainer(user: string) { + private startHelperContainer(user: string) { dockerExec([ 'run', '--name', this.copyContainerName, @@ -670,7 +688,7 @@ export class AssetStagingVolumeCopy { /** * removes the Docker helper container */ - public cleanHelperContainer() { + private cleanHelperContainer() { dockerExec(['rm', this.copyContainerName]); } @@ -678,7 +696,7 @@ export class AssetStagingVolumeCopy { * copy files from the host where this is executed into the input volume * @param sourcePath - path to folder where files should be copied from - without trailing slash */ - public copyInputFrom(sourcePath: string) { + private copyInputFrom(sourcePath: string) { dockerExec(['cp', `${sourcePath}/.`, `${this.copyContainerName}:${AssetStaging.BUNDLING_INPUT_DIR}`]); } @@ -687,7 +705,7 @@ export class AssetStagingVolumeCopy { * copy files from the the output volume to the host where this is executed * @param outputPath - path to folder where files should be copied to - without trailing slash */ - public copyOutputTo(outputPath: string) { + private copyOutputTo(outputPath: string) { dockerExec(['cp', `${this.copyContainerName}:${AssetStaging.BUNDLING_OUTPUT_DIR}/.`, outputPath]); } @@ -695,13 +713,14 @@ export class AssetStagingVolumeCopy { * Bundle files with VOLUME_COPY method */ public run() { + const user = this.getUser(); this.prepareVolumes(); - this.startHelperContainer(this.options.user ?? '1000:1000'); // TODO handle user properly + this.startHelperContainer(user); // TODO handle user properly this.copyInputFrom(this.options.sourcePath); this.options.image.run({ command: this.options.command, - user: this.options.user, + user: user, environment: this.options.environment, entrypoint: this.options.entrypoint, workingDirectory: this.options.workingDirectory ?? AssetStaging.BUNDLING_INPUT_DIR, diff --git a/packages/@aws-cdk/core/test/bundling.test.ts b/packages/@aws-cdk/core/test/bundling.test.ts index 503c886c125d3..2892a3ea1d42c 100644 --- a/packages/@aws-cdk/core/test/bundling.test.ts +++ b/packages/@aws-cdk/core/test/bundling.test.ts @@ -599,7 +599,7 @@ describe('bundling', () => { ], { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); }); - test('DockerImageBundlingCopyHelper prepareVolumes', () => { + test('AssetStagingVolumeCopy bundles with volume copy ', () => { // GIVEN sinon.stub(process, 'platform').value('darwin'); const spawnSyncStub = sinon.stub(child_process, 'spawnSync').returns({ @@ -610,10 +610,16 @@ describe('bundling', () => { output: ['stdout', 'stderr'], signal: null, }); - const helper = new AssetStagingVolumeCopy(); - helper.prepareVolumes(); + const options = { + sourcePath: '/tmp/source', + bundleDir: '/tmp/output', + image: DockerImage.fromRegistry('alpine'), + user: '1000', + }; + const helper = new AssetStagingVolumeCopy(options); + helper.run(); - // THEN + // volume Creation expect(spawnSyncStub.calledWith('docker', sinon.match([ 'volume', 'create', sinon.match(/assetInput.*/g), ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); @@ -622,23 +628,7 @@ describe('bundling', () => { 'volume', 'create', sinon.match(/assetOutput.*/g), ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); - }); - - test('DockerImageBundlingCopyHelper cleanVolumes', () => { - // GIVEN - sinon.stub(process, 'platform').value('darwin'); - const spawnSyncStub = sinon.stub(child_process, 'spawnSync').returns({ - status: 0, - stderr: Buffer.from('stderr'), - stdout: Buffer.from('stdout'), - pid: 123, - output: ['stdout', 'stderr'], - signal: null, - }); - const helper = new AssetStagingVolumeCopy(); - helper.cleanVolumes(); - - // THEN + // volume removal expect(spawnSyncStub.calledWith('docker', sinon.match([ 'volume', 'rm', sinon.match(/assetInput.*/g), ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); @@ -647,24 +637,7 @@ describe('bundling', () => { 'volume', 'rm', sinon.match(/assetOutput.*/g), ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); - }); - - test('DockerImageBundlingCopyHelper startHelperContainer', () => { - // GIVEN - sinon.stub(process, 'platform').value('darwin'); - const spawnSyncStub = sinon.stub(child_process, 'spawnSync').returns({ - status: 0, - stderr: Buffer.from('stderr'), - stdout: Buffer.from('stdout'), - pid: 123, - output: ['stdout', 'stderr'], - signal: null, - }); - const helper = new AssetStagingVolumeCopy(); - const user = '1000'; - helper.startHelperContainer(user); - - // THEN + // prepare copy container expect(spawnSyncStub.calledWith('docker', sinon.match([ 'run', '--name', sinon.match(/copyContainer.*/g), @@ -673,70 +646,21 @@ describe('bundling', () => { 'alpine', 'sh', '-c', - `mkdir -p ${AssetStaging.BUNDLING_INPUT_DIR} && chown -R ${user} ${AssetStaging.BUNDLING_OUTPUT_DIR} && chown -R ${user} ${AssetStaging.BUNDLING_INPUT_DIR}`, + `mkdir -p ${AssetStaging.BUNDLING_INPUT_DIR} && chown -R ${options.user} ${AssetStaging.BUNDLING_OUTPUT_DIR} && chown -R ${options.user} ${AssetStaging.BUNDLING_INPUT_DIR}`, ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); - }); - - test('DockerImageBundlingCopyHelper cleanHelperContainer', () => { - // GIVEN - sinon.stub(process, 'platform').value('darwin'); - const spawnSyncStub = sinon.stub(child_process, 'spawnSync').returns({ - status: 0, - stderr: Buffer.from('stderr'), - stdout: Buffer.from('stdout'), - pid: 123, - output: ['stdout', 'stderr'], - signal: null, - }); - const helper = new AssetStagingVolumeCopy(); - helper.cleanHelperContainer(); - // THEN expect(spawnSyncStub.calledWith('docker', sinon.match([ 'rm', sinon.match(/copyContainer.*/g), ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); - }); - test('DockerImageBundlingCopyHelper copyInputFrom', () => { - // GIVEN - sinon.stub(process, 'platform').value('darwin'); - const spawnSyncStub = sinon.stub(child_process, 'spawnSync').returns({ - status: 0, - stderr: Buffer.from('stderr'), - stdout: Buffer.from('stdout'), - pid: 123, - output: ['stdout', 'stderr'], - signal: null, - }); - const helper = new AssetStagingVolumeCopy(); - const dir = '/test/dir'; - helper.copyInputFrom(dir); - - // THEN expect(spawnSyncStub.calledWith('docker', sinon.match([ - 'cp', `${dir}/.`, `${helper.copyContainerName}:${AssetStaging.BUNDLING_INPUT_DIR}`, + 'cp', `${options.sourcePath}/.`, `${helper.copyContainerName}:${AssetStaging.BUNDLING_INPUT_DIR}`, ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); - }); - test('DockerImageBundlingCopyHelper copyOutputTo', () => { - // GIVEN - sinon.stub(process, 'platform').value('darwin'); - const spawnSyncStub = sinon.stub(child_process, 'spawnSync').returns({ - status: 0, - stderr: Buffer.from('stderr'), - stdout: Buffer.from('stdout'), - pid: 123, - output: ['stdout', 'stderr'], - signal: null, - }); - const helper = new AssetStagingVolumeCopy(); - const dir = '/test/dir'; - helper.copyOutputTo(dir); - - // THEN expect(spawnSyncStub.calledWith('docker', sinon.match([ - 'cp', `${helper.copyContainerName}:${AssetStaging.BUNDLING_OUTPUT_DIR}/.`, dir, + 'cp', `${helper.copyContainerName}:${AssetStaging.BUNDLING_OUTPUT_DIR}/.`, options.bundleDir, ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); + }); }); From ace87a4fefc969aab30f041677c31ae457c33066 Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Mon, 23 Jan 2023 11:17:30 +0100 Subject: [PATCH 22/34] move classes to private --- packages/@aws-cdk/aws-lambda-go/README.md | 2 +- .../@aws-cdk/aws-lambda-go/lib/bundling.ts | 4 +- packages/@aws-cdk/aws-lambda-go/lib/types.ts | 2 +- .../aws-lambda-go/test/bundling.test.ts | 4 +- packages/@aws-cdk/aws-lambda-nodejs/README.md | 2 +- .../aws-lambda-nodejs/lib/bundling.ts | 4 +- .../@aws-cdk/aws-lambda-nodejs/lib/types.ts | 2 +- .../aws-lambda-nodejs/test/bundling.test.ts | 4 +- packages/@aws-cdk/aws-lambda-python/README.md | 2 +- .../aws-lambda-python/lib/bundling.ts | 4 +- .../@aws-cdk/aws-lambda-python/lib/types.ts | 2 +- .../aws-lambda-python/test/bundling.test.ts | 4 +- .../test/integ.function.dockercopy.ts | 2 +- packages/@aws-cdk/core/lib/asset-staging.ts | 4 +- packages/@aws-cdk/core/lib/bundling.ts | 205 +--------------- .../core/lib/private/asset-staging.ts | 221 ++++++++++++++++++ packages/@aws-cdk/core/test/bundling.test.ts | 66 +----- .../core/test/private/asset-staging.test.ts | 115 +++++++++ packages/@aws-cdk/core/test/staging.test.ts | 6 +- 19 files changed, 364 insertions(+), 291 deletions(-) create mode 100644 packages/@aws-cdk/core/lib/private/asset-staging.ts create mode 100644 packages/@aws-cdk/core/test/private/asset-staging.test.ts diff --git a/packages/@aws-cdk/aws-lambda-go/README.md b/packages/@aws-cdk/aws-lambda-go/README.md index 84561b50c52f9..46553a25ec31d 100644 --- a/packages/@aws-cdk/aws-lambda-go/README.md +++ b/packages/@aws-cdk/aws-lambda-go/README.md @@ -295,7 +295,7 @@ import { BundlingFileCopyVariant } from '@aws-cdk/core'; new go.GoFunction(this, 'GoFunction', { entry: 'app/cmd/api', bundling: { - fileCopyVariant: BundlingFileCopyVariant.VOLUME_COPY, + assetStagingType: BundlingFileCopyVariant.VOLUME_COPY, }, }); ``` diff --git a/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts b/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts index abdb021d45db1..3873ec5d5612c 100644 --- a/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts +++ b/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts @@ -66,7 +66,7 @@ export interface BundlingProps extends BundlingOptions { * Which option to use to copy the source files to the docker container and output files back * @default - BundlingFileCopyVariant.BIND_MOUNT */ - readonly fileCopyVariant?: BundlingFileCopyVariant; + readonly assetStagingType?: BundlingFileCopyVariant; } /** @@ -163,7 +163,7 @@ export class Bundling implements cdk.BundlingOptions { this.user = props.user; this.securityOpt = props.securityOpt; this.network = props.network; - this.assetStagingType = props.fileCopyVariant; + this.assetStagingType = props.assetStagingType; // Local bundling if (!props.forcedDockerBundling) { // only if Docker is not forced diff --git a/packages/@aws-cdk/aws-lambda-go/lib/types.ts b/packages/@aws-cdk/aws-lambda-go/lib/types.ts index f7ac7bef9155f..268c7dff17a0c 100644 --- a/packages/@aws-cdk/aws-lambda-go/lib/types.ts +++ b/packages/@aws-cdk/aws-lambda-go/lib/types.ts @@ -116,7 +116,7 @@ export interface BundlingOptions extends DockerRunOptions { * Which option to use to copy the source files to the docker container and output files back * @default - BundlingFileCopyVariant.BIND_MOUNT */ - readonly fileCopyVariant?: BundlingFileCopyVariant; + readonly assetStagingType?: BundlingFileCopyVariant; } /** diff --git a/packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts b/packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts index 0494c55bfe885..e6088e78b1d09 100644 --- a/packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts +++ b/packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts @@ -469,13 +469,13 @@ test('Custom bundling file copy variant', () => { runtime: Runtime.GO_1_X, architecture: Architecture.X86_64, forcedDockerBundling: true, - fileCopyVariant: BundlingFileCopyVariant.VOLUME_COPY, + assetStagingType: BundlingFileCopyVariant.VOLUME_COPY, }); expect(Code.fromAsset).toHaveBeenCalledWith('/project', { assetHashType: AssetHashType.OUTPUT, bundling: expect.objectContaining({ - fileCopyVariant: BundlingFileCopyVariant.VOLUME_COPY, + assetStagingType: BundlingFileCopyVariant.VOLUME_COPY, }), }); }); diff --git a/packages/@aws-cdk/aws-lambda-nodejs/README.md b/packages/@aws-cdk/aws-lambda-nodejs/README.md index 825e57cc367a5..8fb5d3a2a68c8 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/README.md +++ b/packages/@aws-cdk/aws-lambda-nodejs/README.md @@ -347,7 +347,7 @@ In situtations where this does not work, like Docker-in-Docker setups or when us import { BundlingFileCopyVariant } from '@aws-cdk/core'; new nodejs.NodejsFunction(this, 'my-handler', { bundling: { - fileCopyVariant: BundlingFileCopyVariant.VOLUME_COPY, + assetStagingType: BundlingFileCopyVariant.VOLUME_COPY, }, }); ``` diff --git a/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts b/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts index a304ddffcb996..80aaacbda2ce7 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts +++ b/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts @@ -48,7 +48,7 @@ export interface BundlingProps extends BundlingOptions { * Which option to use to copy the source files to the docker container and output files back * @default - BundlingFileCopyVariant.BIND_MOUNT */ - readonly fileCopyVariant?: BundlingFileCopyVariant; + readonly assetStagingType?: BundlingFileCopyVariant; } @@ -162,7 +162,7 @@ export class Bundling implements cdk.BundlingOptions { this.user = props.user; this.securityOpt = props.securityOpt; this.network = props.network; - this.assetStagingType = props.fileCopyVariant; + this.assetStagingType = props.assetStagingType; // Local bundling if (!props.forceDockerBundling) { // only if Docker is not forced diff --git a/packages/@aws-cdk/aws-lambda-nodejs/lib/types.ts b/packages/@aws-cdk/aws-lambda-nodejs/lib/types.ts index 0b5bce1737c7c..e26487d8b7c0f 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/lib/types.ts +++ b/packages/@aws-cdk/aws-lambda-nodejs/lib/types.ts @@ -306,7 +306,7 @@ export interface BundlingOptions extends DockerRunOptions { * Which option to use to copy the source files to the docker container and output files back * @default - BundlingFileCopyVariant.BIND_MOUNT */ - readonly fileCopyVariant?: BundlingFileCopyVariant; + readonly assetStagingType?: BundlingFileCopyVariant; } /** diff --git a/packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts b/packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts index 64909754bff33..a78e533b3a2f4 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts +++ b/packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts @@ -835,13 +835,13 @@ test('Custom bundling file copy variant', () => { runtime: Runtime.NODEJS_14_X, architecture: Architecture.X86_64, forceDockerBundling: true, - fileCopyVariant: BundlingFileCopyVariant.VOLUME_COPY, + assetStagingType: BundlingFileCopyVariant.VOLUME_COPY, }); expect(Code.fromAsset).toHaveBeenCalledWith('/project', { assetHashType: AssetHashType.OUTPUT, bundling: expect.objectContaining({ - fileCopyVariant: BundlingFileCopyVariant.VOLUME_COPY, + assetStagingType: BundlingFileCopyVariant.VOLUME_COPY, }), }); }); diff --git a/packages/@aws-cdk/aws-lambda-python/README.md b/packages/@aws-cdk/aws-lambda-python/README.md index abcd25aaad860..bff11d611f34d 100644 --- a/packages/@aws-cdk/aws-lambda-python/README.md +++ b/packages/@aws-cdk/aws-lambda-python/README.md @@ -266,7 +266,7 @@ new python.PythonFunction(this, 'function', { entry, runtime: Runtime.PYTHON_3_8, bundling: { - fileCopyVariant: BundlingFileCopyVariant.VOLUME_COPY, + assetStagingType: BundlingFileCopyVariant.VOLUME_COPY, }, }); ``` diff --git a/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts b/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts index 3b141c1536577..9b9c09bdf3e05 100644 --- a/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts +++ b/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts @@ -46,7 +46,7 @@ export interface BundlingProps extends BundlingOptions { * Which option to use to copy the source files to the docker container and output files back * @default - BundlingFileCopyVariant.BIND_MOUNT */ - fileCopyVariant?: BundlingFileCopyVariant + assetStagingType?: BundlingFileCopyVariant } /** @@ -111,7 +111,7 @@ export class Bundling implements CdkBundlingOptions { this.user = props.user; this.securityOpt = props.securityOpt; this.network = props.network; - this.assetStagingType = props.fileCopyVariant; + this.assetStagingType = props.assetStagingType; } private createBundlingCommand(options: BundlingCommandOptions): string[] { diff --git a/packages/@aws-cdk/aws-lambda-python/lib/types.ts b/packages/@aws-cdk/aws-lambda-python/lib/types.ts index 1f384fa6c8de0..39add7fd7b4b5 100644 --- a/packages/@aws-cdk/aws-lambda-python/lib/types.ts +++ b/packages/@aws-cdk/aws-lambda-python/lib/types.ts @@ -91,7 +91,7 @@ export interface BundlingOptions extends DockerRunOptions { * Which option to use to copy the source files to the docker container and output files back * @default - BundlingFileCopyVariant.BIND_MOUNT */ - readonly fileCopyVariant?: BundlingFileCopyVariant; + readonly assetStagingType?: BundlingFileCopyVariant; } /** diff --git a/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts b/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts index ef7bd5b9950d7..79ee41739fb87 100644 --- a/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts +++ b/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts @@ -379,13 +379,13 @@ test('Bundling with docker copy variant', () => { Bundling.bundle({ entry: entry, runtime: Runtime.PYTHON_3_7, - fileCopyVariant: BundlingFileCopyVariant.VOLUME_COPY, + assetStagingType: BundlingFileCopyVariant.VOLUME_COPY, }); expect(Code.fromAsset).toHaveBeenCalledWith(entry, expect.objectContaining({ bundling: expect.objectContaining({ - fileCopyVariant: BundlingFileCopyVariant.VOLUME_COPY, + assetStagingType: BundlingFileCopyVariant.VOLUME_COPY, }), })); }); diff --git a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.ts b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.ts index d66263e72913d..2aa4e91b8daf4 100644 --- a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.ts +++ b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.ts @@ -22,7 +22,7 @@ class TestStack extends Stack { entry: path.join(__dirname, 'lambda-handler-dockercopy'), runtime: Runtime.PYTHON_3_9, bundling: { - fileCopyVariant: BundlingFileCopyVariant.VOLUME_COPY, + assetStagingType: BundlingFileCopyVariant.VOLUME_COPY, }, }); this.functionName = fn.functionName; diff --git a/packages/@aws-cdk/core/lib/asset-staging.ts b/packages/@aws-cdk/core/lib/asset-staging.ts index a9e71e7360d91..d9ffdfd4f2a7c 100644 --- a/packages/@aws-cdk/core/lib/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/asset-staging.ts @@ -4,10 +4,11 @@ import * as cxapi from '@aws-cdk/cx-api'; import { Construct } from 'constructs'; import * as fs from 'fs-extra'; import { AssetHashType, AssetOptions, FileAssetPackaging } from './assets'; -import { BundlingFileCopyVariant, BundlingOptions, BundlingOutput, AssetStagingVolumeCopy, AssetStagingBindMount } from './bundling'; +import { BundlingFileCopyVariant, BundlingOptions, BundlingOutput } from './bundling'; import { FileSystem, FingerprintOptions } from './fs'; import { clearLargeFileFingerprintCache } from './fs/fingerprint'; import { Names } from './names'; +import { AssetStagingVolumeCopy, AssetStagingBindMount } from './private/asset-staging'; import { Cache } from './private/cache'; import { Stack } from './stack'; import { Stage } from './stage'; @@ -622,3 +623,4 @@ function getExtension(source: string): string { return path.extname(source); }; + diff --git a/packages/@aws-cdk/core/lib/bundling.ts b/packages/@aws-cdk/core/lib/bundling.ts index e516a70d38ac8..3e3082271c19a 100644 --- a/packages/@aws-cdk/core/lib/bundling.ts +++ b/packages/@aws-cdk/core/lib/bundling.ts @@ -1,9 +1,8 @@ -import { spawnSync, SpawnSyncOptions } from 'child_process'; +import { spawnSync } from 'child_process'; import * as crypto from 'crypto'; -import * as os from 'os'; import { isAbsolute, join } from 'path'; -import { AssetStaging } from './asset-staging'; import { FileSystem } from './fs'; +import { dockerExec } from './private/asset-staging'; import { quiet, reset } from './private/jsii-deprecated'; /** @@ -560,210 +559,10 @@ export interface DockerBuildOptions { readonly targetStage?: string; } -/** - * Options for Docker based bundling of assets - */ -export interface AssetStagingCopyOptions extends BundlingOptions { - /** - * Path where the source files are located - */ - readonly sourcePath: string; - /** - * Path where the output files should be stored - */ - readonly bundleDir: string; -} - -class AssetStagingBase { - protected options: AssetStagingCopyOptions - constructor(options: AssetStagingCopyOptions) { - this.options = options; - } - /** - * Determines a useful default user if not given otherwise - */ - protected getUser() { - let user: string; - if (this.options.user) { - user = this.options.user; - } else { // Default to current user - const userInfo = os.userInfo(); - user = userInfo.uid !== -1 // uid is -1 on Windows - ? `${userInfo.uid}:${userInfo.gid}` - : '1000:1000'; - } - return user; - } -} - -/** - * Bundles files with bind mount as copy method - */ -export class AssetStagingBindMount extends AssetStagingBase { - /** - * Bundle files with bind mount as copy method - */ - public run() { - this.options.image.run({ - command: this.options.command, - user: this.getUser(), - environment: this.options.environment, - entrypoint: this.options.entrypoint, - workingDirectory: this.options.workingDirectory ?? AssetStaging.BUNDLING_INPUT_DIR, - securityOpt: this.options.securityOpt ?? '', - volumesFrom: this.options.volumesFrom, - volumes: [ - { - hostPath: this.options.sourcePath, - containerPath: AssetStaging.BUNDLING_INPUT_DIR, - }, - { - hostPath: this.options.bundleDir, - containerPath: AssetStaging.BUNDLING_OUTPUT_DIR, - }, - ...this.options.volumes ?? [], - ], - }); - } -} - -/** - * Provides a helper container for copying bundling related files to specific input and output volumes - */ -export class AssetStagingVolumeCopy extends AssetStagingBase { - /** - * Name of the Docker volume that is used for the asset input - */ - private inputVolumeName: string; - /** - * Name of the Docker volume that is used for the asset output - */ - private outputVolumeName: string; - /** - * Name of the Docker helper container to copy files into the volume - */ - public copyContainerName: string; - - constructor(options: AssetStagingCopyOptions) { - super(options); - const copySuffix = crypto.randomBytes(12).toString('hex'); - this.inputVolumeName = `assetInput${copySuffix}`; - this.outputVolumeName = `assetOutput${copySuffix}`; - this.copyContainerName = `copyContainer${copySuffix}`; - } - - /** - * Creates volumes for asset input and output - */ - private prepareVolumes() { - dockerExec(['volume', 'create', this.inputVolumeName]); - dockerExec(['volume', 'create', this.outputVolumeName]); - } - - /** - * Removes volumes for asset input and output - */ - private cleanVolumes() { - dockerExec(['volume', 'rm', this.inputVolumeName]); - dockerExec(['volume', 'rm', this.outputVolumeName]); - } - - /** - * runs a helper container that holds volumes and does some preparation tasks - * @param user The user that will later access these files and needs permissions to do so - */ - private startHelperContainer(user: string) { - dockerExec([ - 'run', - '--name', this.copyContainerName, - '-v', `${this.inputVolumeName}:${AssetStaging.BUNDLING_INPUT_DIR}`, - '-v', `${this.outputVolumeName}:${AssetStaging.BUNDLING_OUTPUT_DIR}`, - 'alpine', - 'sh', - '-c', - `mkdir -p ${AssetStaging.BUNDLING_INPUT_DIR} && chown -R ${user} ${AssetStaging.BUNDLING_OUTPUT_DIR} && chown -R ${user} ${AssetStaging.BUNDLING_INPUT_DIR}`, - ]); - } - - /** - * removes the Docker helper container - */ - private cleanHelperContainer() { - dockerExec(['rm', this.copyContainerName]); - } - - /** - * copy files from the host where this is executed into the input volume - * @param sourcePath - path to folder where files should be copied from - without trailing slash - */ - private copyInputFrom(sourcePath: string) { - dockerExec(['cp', `${sourcePath}/.`, `${this.copyContainerName}:${AssetStaging.BUNDLING_INPUT_DIR}`]); - - } - - /** - * copy files from the the output volume to the host where this is executed - * @param outputPath - path to folder where files should be copied to - without trailing slash - */ - private copyOutputTo(outputPath: string) { - dockerExec(['cp', `${this.copyContainerName}:${AssetStaging.BUNDLING_OUTPUT_DIR}/.`, outputPath]); - } - - /** - * Bundle files with VOLUME_COPY method - */ - public run() { - const user = this.getUser(); - this.prepareVolumes(); - this.startHelperContainer(user); // TODO handle user properly - this.copyInputFrom(this.options.sourcePath); - - this.options.image.run({ - command: this.options.command, - user: user, - environment: this.options.environment, - entrypoint: this.options.entrypoint, - workingDirectory: this.options.workingDirectory ?? AssetStaging.BUNDLING_INPUT_DIR, - securityOpt: this.options.securityOpt ?? '', - volumes: this.options.volumes, - volumesFrom: [this.copyContainerName, ...this.options.volumesFrom ?? []], - }); - - this.copyOutputTo(this.options.bundleDir); - this.cleanHelperContainer(); - this.cleanVolumes(); - } - -} - function flatten(x: string[][]) { return Array.prototype.concat([], ...x); } -function dockerExec(args: string[], options?: SpawnSyncOptions) { - const prog = process.env.CDK_DOCKER ?? 'docker'; - const proc = spawnSync(prog, args, options ?? { - stdio: [ // show Docker output - 'ignore', // ignore stdio - process.stderr, // redirect stdout to stderr - 'inherit', // inherit stderr - ], - }); - - if (proc.error) { - throw proc.error; - } - - if (proc.status !== 0) { - if (proc.stdout || proc.stderr) { - throw new Error(`[Status ${proc.status}] stdout: ${proc.stdout?.toString().trim()}\n\n\nstderr: ${proc.stderr?.toString().trim()}`); - } - throw new Error(`${prog} exited with status ${proc.status}`); - } - - return proc; -} - function isSeLinux() : boolean { if (process.platform != 'linux') { return false; diff --git a/packages/@aws-cdk/core/lib/private/asset-staging.ts b/packages/@aws-cdk/core/lib/private/asset-staging.ts new file mode 100644 index 0000000000000..b80724036f20d --- /dev/null +++ b/packages/@aws-cdk/core/lib/private/asset-staging.ts @@ -0,0 +1,221 @@ +import { spawnSync, SpawnSyncOptions } from 'child_process'; +import * as crypto from 'crypto'; +import * as os from 'os'; +import { AssetStaging } from '../asset-staging'; +import { BundlingOptions } from '../bundling'; + +/** + * Options for Docker based bundling of assets + */ +interface AssetStagingCopyOptions extends BundlingOptions { + /** + * Path where the source files are located + */ + readonly sourcePath: string; + /** + * Path where the output files should be stored + */ + readonly bundleDir: string; +} + +class AssetStagingBase { + protected options: AssetStagingCopyOptions; + constructor(options: AssetStagingCopyOptions) { + this.options = options; + } + /** + * Determines a useful default user if not given otherwise + */ + protected determineUser() { + let user: string; + if (this.options.user) { + user = this.options.user; + } else { + // Default to current user + const userInfo = os.userInfo(); + user = + userInfo.uid !== -1 // uid is -1 on Windows + ? `${userInfo.uid}:${userInfo.gid}` + : '1000:1000'; + } + return user; + } +} + +/** + * Bundles files with bind mount as copy method + */ +export class AssetStagingBindMount extends AssetStagingBase { + /** + * Bundle files with bind mount as copy method + */ + public run() { + this.options.image.run({ + command: this.options.command, + user: this.determineUser(), + environment: this.options.environment, + entrypoint: this.options.entrypoint, + workingDirectory: + this.options.workingDirectory ?? AssetStaging.BUNDLING_INPUT_DIR, + securityOpt: this.options.securityOpt ?? '', + volumesFrom: this.options.volumesFrom, + volumes: [ + { + hostPath: this.options.sourcePath, + containerPath: AssetStaging.BUNDLING_INPUT_DIR, + }, + { + hostPath: this.options.bundleDir, + containerPath: AssetStaging.BUNDLING_OUTPUT_DIR, + }, + ...(this.options.volumes ?? []), + ], + }); + } +} + +/** + * Provides a helper container for copying bundling related files to specific input and output volumes + */ +export class AssetStagingVolumeCopy extends AssetStagingBase { + /** + * Name of the Docker volume that is used for the asset input + */ + private inputVolumeName: string; + /** + * Name of the Docker volume that is used for the asset output + */ + private outputVolumeName: string; + /** + * Name of the Docker helper container to copy files into the volume + */ + public copyContainerName: string; + + constructor(options: AssetStagingCopyOptions) { + super(options); + const copySuffix = crypto.randomBytes(12).toString('hex'); + this.inputVolumeName = `assetInput${copySuffix}`; + this.outputVolumeName = `assetOutput${copySuffix}`; + this.copyContainerName = `copyContainer${copySuffix}`; + } + + /** + * Creates volumes for asset input and output + */ + private prepareVolumes() { + dockerExec(['volume', 'create', this.inputVolumeName]); + dockerExec(['volume', 'create', this.outputVolumeName]); + } + + /** + * Removes volumes for asset input and output + */ + private cleanVolumes() { + dockerExec(['volume', 'rm', this.inputVolumeName]); + dockerExec(['volume', 'rm', this.outputVolumeName]); + } + + /** + * runs a helper container that holds volumes and does some preparation tasks + * @param user The user that will later access these files and needs permissions to do so + */ + private startHelperContainer(user: string) { + dockerExec([ + 'run', + '--name', + this.copyContainerName, + '-v', + `${this.inputVolumeName}:${AssetStaging.BUNDLING_INPUT_DIR}`, + '-v', + `${this.outputVolumeName}:${AssetStaging.BUNDLING_OUTPUT_DIR}`, + 'alpine', + 'sh', + '-c', + `mkdir -p ${AssetStaging.BUNDLING_INPUT_DIR} && chown -R ${user} ${AssetStaging.BUNDLING_OUTPUT_DIR} && chown -R ${user} ${AssetStaging.BUNDLING_INPUT_DIR}`, + ]); + } + + /** + * removes the Docker helper container + */ + private cleanHelperContainer() { + dockerExec(['rm', this.copyContainerName]); + } + + /** + * copy files from the host where this is executed into the input volume + * @param sourcePath - path to folder where files should be copied from - without trailing slash + */ + private copyInputFrom(sourcePath: string) { + dockerExec([ + 'cp', + `${sourcePath}/.`, + `${this.copyContainerName}:${AssetStaging.BUNDLING_INPUT_DIR}`, + ]); + } + + /** + * copy files from the the output volume to the host where this is executed + * @param outputPath - path to folder where files should be copied to - without trailing slash + */ + private copyOutputTo(outputPath: string) { + dockerExec([ + 'cp', + `${this.copyContainerName}:${AssetStaging.BUNDLING_OUTPUT_DIR}/.`, + outputPath, + ]); + } + + /** + * Bundle files with VOLUME_COPY method + */ + public run() { + const user = this.determineUser(); + this.prepareVolumes(); + this.startHelperContainer(user); // TODO handle user properly + this.copyInputFrom(this.options.sourcePath); + + this.options.image.run({ + command: this.options.command, + user: user, + environment: this.options.environment, + entrypoint: this.options.entrypoint, + workingDirectory: + this.options.workingDirectory ?? AssetStaging.BUNDLING_INPUT_DIR, + securityOpt: this.options.securityOpt ?? '', + volumes: this.options.volumes, + volumesFrom: [ + this.copyContainerName, + ...(this.options.volumesFrom ?? []), + ], + }); + + this.copyOutputTo(this.options.bundleDir); + this.cleanHelperContainer(); + this.cleanVolumes(); + } +} + +export function dockerExec(args: string[], options?: SpawnSyncOptions) { + const prog = process.env.CDK_DOCKER ?? 'docker'; + const proc = spawnSync(prog, args, options ?? { + stdio: [ // show Docker output + 'ignore', // ignore stdio + process.stderr, // redirect stdout to stderr + 'inherit', // inherit stderr + ], + }); + + if (proc.error) { + throw proc.error; + } + + if (proc.status !== 0) { + if (proc.stdout || proc.stderr) { + throw new Error(`[Status ${proc.status}] stdout: ${proc.stdout?.toString().trim()}\n\n\nstderr: ${proc.stderr?.toString().trim()}`); + } + throw new Error(`${prog} exited with status ${proc.status}`); + } + + return proc; +} \ No newline at end of file diff --git a/packages/@aws-cdk/core/test/bundling.test.ts b/packages/@aws-cdk/core/test/bundling.test.ts index 2892a3ea1d42c..4b364020b97dd 100644 --- a/packages/@aws-cdk/core/test/bundling.test.ts +++ b/packages/@aws-cdk/core/test/bundling.test.ts @@ -2,7 +2,7 @@ import * as child_process from 'child_process'; import * as crypto from 'crypto'; import * as path from 'path'; import * as sinon from 'sinon'; -import { AssetStaging, DockerImage, AssetStagingVolumeCopy, FileSystem } from '../lib'; +import { DockerImage, FileSystem } from '../lib'; describe('bundling', () => { afterEach(() => { @@ -599,68 +599,4 @@ describe('bundling', () => { ], { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); }); - test('AssetStagingVolumeCopy bundles with volume copy ', () => { - // GIVEN - sinon.stub(process, 'platform').value('darwin'); - const spawnSyncStub = sinon.stub(child_process, 'spawnSync').returns({ - status: 0, - stderr: Buffer.from('stderr'), - stdout: Buffer.from('stdout'), - pid: 123, - output: ['stdout', 'stderr'], - signal: null, - }); - const options = { - sourcePath: '/tmp/source', - bundleDir: '/tmp/output', - image: DockerImage.fromRegistry('alpine'), - user: '1000', - }; - const helper = new AssetStagingVolumeCopy(options); - helper.run(); - - // volume Creation - expect(spawnSyncStub.calledWith('docker', sinon.match([ - 'volume', 'create', sinon.match(/assetInput.*/g), - ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); - - expect(spawnSyncStub.calledWith('docker', sinon.match([ - 'volume', 'create', sinon.match(/assetOutput.*/g), - ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); - - // volume removal - expect(spawnSyncStub.calledWith('docker', sinon.match([ - 'volume', 'rm', sinon.match(/assetInput.*/g), - ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); - - expect(spawnSyncStub.calledWith('docker', sinon.match([ - 'volume', 'rm', sinon.match(/assetOutput.*/g), - ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); - - // prepare copy container - expect(spawnSyncStub.calledWith('docker', sinon.match([ - 'run', - '--name', sinon.match(/copyContainer.*/g), - '-v', sinon.match(/assetInput.*/g), - '-v', sinon.match(/assetOutput.*/g), - 'alpine', - 'sh', - '-c', - `mkdir -p ${AssetStaging.BUNDLING_INPUT_DIR} && chown -R ${options.user} ${AssetStaging.BUNDLING_OUTPUT_DIR} && chown -R ${options.user} ${AssetStaging.BUNDLING_INPUT_DIR}`, - ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); - - expect(spawnSyncStub.calledWith('docker', sinon.match([ - 'rm', sinon.match(/copyContainer.*/g), - ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); - - expect(spawnSyncStub.calledWith('docker', sinon.match([ - 'cp', `${options.sourcePath}/.`, `${helper.copyContainerName}:${AssetStaging.BUNDLING_INPUT_DIR}`, - ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); - - expect(spawnSyncStub.calledWith('docker', sinon.match([ - 'cp', `${helper.copyContainerName}:${AssetStaging.BUNDLING_OUTPUT_DIR}/.`, options.bundleDir, - ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); - - }); - }); diff --git a/packages/@aws-cdk/core/test/private/asset-staging.test.ts b/packages/@aws-cdk/core/test/private/asset-staging.test.ts new file mode 100644 index 0000000000000..825b78bd59e35 --- /dev/null +++ b/packages/@aws-cdk/core/test/private/asset-staging.test.ts @@ -0,0 +1,115 @@ +import * as child_process from 'child_process'; +import * as sinon from 'sinon'; +import { AssetStaging, DockerImage } from '../../lib'; +import { AssetStagingBindMount, AssetStagingVolumeCopy } from '../../lib/private/asset-staging'; + +describe('bundling', () => { + afterEach(() => { + sinon.restore(); + + }); + + test('AssetStagingVolumeCopy bundles with volume copy ', () => { + // GIVEN + sinon.stub(process, 'platform').value('darwin'); + const spawnSyncStub = sinon.stub(child_process, 'spawnSync').returns({ + status: 0, + stderr: Buffer.from('stderr'), + stdout: Buffer.from('stdout'), + pid: 123, + output: ['stdout', 'stderr'], + signal: null, + }); + const options = { + sourcePath: '/tmp/source', + bundleDir: '/tmp/output', + image: DockerImage.fromRegistry('alpine'), + user: '1000', + }; + const helper = new AssetStagingVolumeCopy(options); + helper.run(); + + // volume Creation + expect(spawnSyncStub.calledWith('docker', sinon.match([ + 'volume', 'create', sinon.match(/assetInput.*/g), + ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); + + expect(spawnSyncStub.calledWith('docker', sinon.match([ + 'volume', 'create', sinon.match(/assetOutput.*/g), + ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); + + // volume removal + expect(spawnSyncStub.calledWith('docker', sinon.match([ + 'volume', 'rm', sinon.match(/assetInput.*/g), + ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); + + expect(spawnSyncStub.calledWith('docker', sinon.match([ + 'volume', 'rm', sinon.match(/assetOutput.*/g), + ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); + + // prepare copy container + expect(spawnSyncStub.calledWith('docker', sinon.match([ + 'run', + '--name', sinon.match(/copyContainer.*/g), + '-v', sinon.match(/assetInput.*/g), + '-v', sinon.match(/assetOutput.*/g), + 'alpine', + 'sh', + '-c', + `mkdir -p ${AssetStaging.BUNDLING_INPUT_DIR} && chown -R ${options.user} ${AssetStaging.BUNDLING_OUTPUT_DIR} && chown -R ${options.user} ${AssetStaging.BUNDLING_INPUT_DIR}`, + ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); + + // delete copy container + expect(spawnSyncStub.calledWith('docker', sinon.match([ + 'rm', sinon.match(/copyContainer.*/g), + ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); + + // copy files to copy container + expect(spawnSyncStub.calledWith('docker', sinon.match([ + 'cp', `${options.sourcePath}/.`, `${helper.copyContainerName}:${AssetStaging.BUNDLING_INPUT_DIR}`, + ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); + + // copy files from copy container to host + expect(spawnSyncStub.calledWith('docker', sinon.match([ + 'cp', `${helper.copyContainerName}:${AssetStaging.BUNDLING_OUTPUT_DIR}/.`, options.bundleDir, + ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); + + // actual docker run + expect(spawnSyncStub.calledWith('docker', sinon.match.array.contains([ + 'run', '--rm', + '--volumes-from', helper.copyContainerName, + 'alpine', + ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); + + }); + + test('AssetStagingBindMount bundles with bind mount ', () => { + // GIVEN + sinon.stub(process, 'platform').value('darwin'); + const spawnSyncStub = sinon.stub(child_process, 'spawnSync').returns({ + status: 0, + stderr: Buffer.from('stderr'), + stdout: Buffer.from('stdout'), + pid: 123, + output: ['stdout', 'stderr'], + signal: null, + }); + const options = { + sourcePath: '/tmp/source', + bundleDir: '/tmp/output', + image: DockerImage.fromRegistry('alpine'), + user: '1000', + }; + const helper = new AssetStagingBindMount(options); + helper.run(); + + // actual docker run with bind mount is called + expect(spawnSyncStub.calledWith('docker', sinon.match.array.contains([ + 'run', '--rm', + '-v', + 'alpine', + ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); + + }); + +}); diff --git a/packages/@aws-cdk/core/test/staging.test.ts b/packages/@aws-cdk/core/test/staging.test.ts index 6491eeb0fa4c9..0da4884647bff 100644 --- a/packages/@aws-cdk/core/test/staging.test.ts +++ b/packages/@aws-cdk/core/test/staging.test.ts @@ -1300,14 +1300,14 @@ describe('staging with docker cp', () => { // THEN const assembly = app.synth(); expect(fs.readdirSync(assembly.directory)).toEqual([ - 'asset.115de8f71f52dcd3958586818e7e8e42be04db3039bc14c0501e0d26efbd653a', // this is the bundle dir - 'asset.115de8f71f52dcd3958586818e7e8e42be04db3039bc14c0501e0d26efbd653a.zip', + 'asset.812ead30bdacbf9ae61ac91c52e460755a9b072440e703620ab7f8cd09eaa78f', // this is the bundle dir + 'asset.812ead30bdacbf9ae61ac91c52e460755a9b072440e703620ab7f8cd09eaa78f.zip', 'cdk.out', 'manifest.json', 'stack.template.json', 'tree.json', ]); - expect(fs.readdirSync(path.join(assembly.directory, 'asset.115de8f71f52dcd3958586818e7e8e42be04db3039bc14c0501e0d26efbd653a'))).toEqual([ + expect(fs.readdirSync(path.join(assembly.directory, 'asset.812ead30bdacbf9ae61ac91c52e460755a9b072440e703620ab7f8cd09eaa78f'))).toEqual([ 'test.zip', // bundle dir with "touched" bundled output file ]); expect(staging.packaging).toEqual(FileAssetPackaging.FILE); From 7be1cc521e83017b1d4dcb4b3678263ce8d6b5bc Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Wed, 25 Jan 2023 13:25:38 +0100 Subject: [PATCH 23/34] Update packages/@aws-cdk/core/test/docker-stub-cp.sh Co-authored-by: Momo Kornher --- packages/@aws-cdk/core/test/docker-stub-cp.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/@aws-cdk/core/test/docker-stub-cp.sh b/packages/@aws-cdk/core/test/docker-stub-cp.sh index 971b3185fabb3..22853577174b9 100755 --- a/packages/@aws-cdk/core/test/docker-stub-cp.sh +++ b/packages/@aws-cdk/core/test/docker-stub-cp.sh @@ -3,8 +3,8 @@ set -euo pipefail # stub for the `docker` executable. it is used as CDK_DOCKER when executing unit # tests in `test.staging.ts` This variant is specific for tests that use the docker copy method for files, instead of bind mounts -echo "$@" >> /tmp/docker-stub.input.concat -echo "$@" > /tmp/docker-stub.input +echo "$@" >> /tmp/docker-stub-cp.input.concat +echo "$@" > /tmp/docker-stub-cp.input # create a fake zip to emulate created files, fetch the target path from the "docker cp" command if echo "$@" | grep "cp"| grep "/asset-output"; then From fa11118b2acf9b267b81e019b4a1ecd22bfffe49 Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Wed, 25 Jan 2023 13:25:56 +0100 Subject: [PATCH 24/34] Update packages/@aws-cdk/core/test/bundling.test.ts Co-authored-by: Momo Kornher --- packages/@aws-cdk/core/test/bundling.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/@aws-cdk/core/test/bundling.test.ts b/packages/@aws-cdk/core/test/bundling.test.ts index 4b364020b97dd..3879d0b27fc41 100644 --- a/packages/@aws-cdk/core/test/bundling.test.ts +++ b/packages/@aws-cdk/core/test/bundling.test.ts @@ -598,5 +598,4 @@ describe('bundling', () => { 'cool', 'command', ], { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); }); - }); From 381e5bf5d3b6cc13be2963544c2bf2b729419cf1 Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Wed, 25 Jan 2023 13:26:10 +0100 Subject: [PATCH 25/34] Update packages/@aws-cdk/core/lib/private/asset-staging.ts Co-authored-by: Momo Kornher --- packages/@aws-cdk/core/lib/private/asset-staging.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/core/lib/private/asset-staging.ts b/packages/@aws-cdk/core/lib/private/asset-staging.ts index b80724036f20d..9d2ac8a33f96f 100644 --- a/packages/@aws-cdk/core/lib/private/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/private/asset-staging.ts @@ -218,4 +218,4 @@ export function dockerExec(args: string[], options?: SpawnSyncOptions) { } return proc; -} \ No newline at end of file +} From 5a9b75eb091dbbbdcb49caff4bffb7365103abb6 Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Wed, 25 Jan 2023 13:26:17 +0100 Subject: [PATCH 26/34] Update packages/@aws-cdk/core/lib/bundling.ts Co-authored-by: Momo Kornher --- packages/@aws-cdk/core/lib/bundling.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/@aws-cdk/core/lib/bundling.ts b/packages/@aws-cdk/core/lib/bundling.ts index 3e3082271c19a..c3325fe0768d5 100644 --- a/packages/@aws-cdk/core/lib/bundling.ts +++ b/packages/@aws-cdk/core/lib/bundling.ts @@ -409,7 +409,6 @@ export class DockerImage extends BundlingDockerImage { reset(deprecated); return result; } - } /** From 8640305efea70de7ae67be40d3079bab678267d6 Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Wed, 25 Jan 2023 13:41:10 +0100 Subject: [PATCH 27/34] allow use of different stub files --- packages/@aws-cdk/core/test/docker-stub-cp.sh | 1 - packages/@aws-cdk/core/test/staging.test.ts | 21 +++++++++++-------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/packages/@aws-cdk/core/test/docker-stub-cp.sh b/packages/@aws-cdk/core/test/docker-stub-cp.sh index 22853577174b9..fec4009896d63 100755 --- a/packages/@aws-cdk/core/test/docker-stub-cp.sh +++ b/packages/@aws-cdk/core/test/docker-stub-cp.sh @@ -10,7 +10,6 @@ echo "$@" > /tmp/docker-stub-cp.input if echo "$@" | grep "cp"| grep "/asset-output"; then outdir=$(echo "$@" | grep cp | grep "/asset-output" | xargs -n1 | grep "cdk.out" | head -n1 | cut -d":" -f1) if [ -n "$outdir" ]; then - echo "${outdir}" >> /tmp/docker-stub.input.concat.output.txt touch "${outdir}/test.zip" fi fi diff --git a/packages/@aws-cdk/core/test/staging.test.ts b/packages/@aws-cdk/core/test/staging.test.ts index 0da4884647bff..8db07a2f94579 100644 --- a/packages/@aws-cdk/core/test/staging.test.ts +++ b/packages/@aws-cdk/core/test/staging.test.ts @@ -11,6 +11,9 @@ import { App, AssetHashType, AssetStaging, DockerImage, BundlingOptions, Bundlin const STUB_INPUT_FILE = '/tmp/docker-stub.input'; const STUB_INPUT_CONCAT_FILE = '/tmp/docker-stub.input.concat'; +const STUB_INPUT_CP_FILE = '/tmp/docker-stub-cp.input'; +const STUB_INPUT_CP_CONCAT_FILE = '/tmp/docker-stub-cp.input.concat'; + enum DockerStubCommand { SUCCESS = 'DOCKER_STUB_SUCCESS', FAIL = 'DOCKER_STUB_FAIL', @@ -1272,11 +1275,11 @@ describe('staging with docker cp', () => { afterEach(() => { AssetStaging.clearAssetHashCache(); - if (fs.existsSync(STUB_INPUT_FILE)) { - fs.unlinkSync(STUB_INPUT_FILE); + if (fs.existsSync(STUB_INPUT_CP_FILE)) { + fs.unlinkSync(STUB_INPUT_CP_FILE); } - if (fs.existsSync(STUB_INPUT_CONCAT_FILE)) { - fs.unlinkSync(STUB_INPUT_CONCAT_FILE); + if (fs.existsSync(STUB_INPUT_CP_CONCAT_FILE)) { + fs.unlinkSync(STUB_INPUT_CP_CONCAT_FILE); } sinon.restore(); }); @@ -1312,7 +1315,7 @@ describe('staging with docker cp', () => { ]); expect(staging.packaging).toEqual(FileAssetPackaging.FILE); expect(staging.isArchive).toEqual(true); - const dockerCalls: string[] = readDockerStubInputConcat().split(/\r?\n/); + const dockerCalls: string[] = readDockerStubInputConcat(STUB_INPUT_CP_CONCAT_FILE).split(/\r?\n/); expect(dockerCalls).toEqual(expect.arrayContaining([ expect.stringContaining('volume create assetInput'), expect.stringContaining('volume create assetOutput'), @@ -1337,10 +1340,10 @@ function readAndCleanDockerStubInput(file: string) { } // Last docker input since last teardown -function readDockerStubInput() { - return readAndCleanDockerStubInput(STUB_INPUT_FILE); +function readDockerStubInput(file?: string) { + return readAndCleanDockerStubInput(file ?? STUB_INPUT_FILE); } // Concatenated docker inputs since last teardown -function readDockerStubInputConcat() { - return readAndCleanDockerStubInput(STUB_INPUT_CONCAT_FILE); +function readDockerStubInputConcat(file?: string) { + return readAndCleanDockerStubInput(file ?? STUB_INPUT_CONCAT_FILE); } From 4750a5f3b88fa0a5805351e738baa343a0cc8a27 Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Wed, 25 Jan 2023 16:27:47 +0100 Subject: [PATCH 28/34] Update packages/@aws-cdk/core/lib/bundling.ts Co-authored-by: Momo Kornher --- packages/@aws-cdk/core/lib/bundling.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/core/lib/bundling.ts b/packages/@aws-cdk/core/lib/bundling.ts index c3325fe0768d5..aaa40e786f9f9 100644 --- a/packages/@aws-cdk/core/lib/bundling.ts +++ b/packages/@aws-cdk/core/lib/bundling.ts @@ -163,7 +163,7 @@ export interface ILocalBundling { */ export enum BundlingFileCopyVariant { /** - * Creates temporary volumes and docker containers + * Creates temporary volumes and containers to copy files from the host to the bundling container and back. * This is slower, but works also in more complex situations with remote or shared docker sockets. */ VOLUME_COPY = 'VOLUME_COPY', From 27026e5e3cc7fd8dbb6310d63c7597a836f7ec20 Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Wed, 25 Jan 2023 16:40:23 +0100 Subject: [PATCH 29/34] change naming --- packages/@aws-cdk/aws-lambda-go/README.md | 4 ++-- packages/@aws-cdk/aws-lambda-go/lib/bundling.ts | 8 ++++---- packages/@aws-cdk/aws-lambda-go/lib/types.ts | 6 +++--- .../@aws-cdk/aws-lambda-go/test/bundling.test.ts | 6 +++--- packages/@aws-cdk/aws-lambda-nodejs/README.md | 4 ++-- .../@aws-cdk/aws-lambda-nodejs/lib/bundling.ts | 8 ++++---- packages/@aws-cdk/aws-lambda-nodejs/lib/types.ts | 6 +++--- .../aws-lambda-nodejs/test/bundling.test.ts | 6 +++--- packages/@aws-cdk/aws-lambda-python/README.md | 4 ++-- .../@aws-cdk/aws-lambda-python/lib/bundling.ts | 8 ++++---- packages/@aws-cdk/aws-lambda-python/lib/types.ts | 6 +++--- .../aws-lambda-python/test/bundling.test.ts | 6 +++--- .../test/integ.function.dockercopy.ts | 4 ++-- packages/@aws-cdk/core/lib/asset-staging.ts | 12 ++++++------ packages/@aws-cdk/core/lib/bundling.ts | 8 +++----- .../@aws-cdk/core/lib/private/asset-staging.ts | 14 +++++++------- .../core/test/private/asset-staging.test.ts | 10 +++++----- packages/@aws-cdk/core/test/staging.test.ts | 4 ++-- 18 files changed, 61 insertions(+), 63 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-go/README.md b/packages/@aws-cdk/aws-lambda-go/README.md index 46553a25ec31d..a11da09adcb60 100644 --- a/packages/@aws-cdk/aws-lambda-go/README.md +++ b/packages/@aws-cdk/aws-lambda-go/README.md @@ -291,11 +291,11 @@ By default the input and output of Docker based bundling is handled via bind mou In situtations where this does not work, like Docker-in-Docker setups or when using a remote Docker socket, you can configure an alternative, but slower, variant that also works in these situations. ```ts -import { BundlingFileCopyVariant } from '@aws-cdk/core'; +import { BundlingFileAccess } from '@aws-cdk/core'; new go.GoFunction(this, 'GoFunction', { entry: 'app/cmd/api', bundling: { - assetStagingType: BundlingFileCopyVariant.VOLUME_COPY, + assetStagingType: BundlingFileAccess.VOLUME_COPY, }, }); ``` diff --git a/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts b/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts index 3873ec5d5612c..3a7f21e49928f 100644 --- a/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts +++ b/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts @@ -2,7 +2,7 @@ import * as os from 'os'; import * as path from 'path'; import { Architecture, AssetCode, Code, Runtime } from '@aws-cdk/aws-lambda'; import * as cdk from '@aws-cdk/core'; -import { BundlingFileCopyVariant } from '@aws-cdk/core'; +import { BundlingFileAccess } from '@aws-cdk/core'; import { BundlingOptions } from './types'; import { exec, findUp, getGoBuildVersion } from './util'; @@ -64,9 +64,9 @@ export interface BundlingProps extends BundlingOptions { /** * Which option to use to copy the source files to the docker container and output files back - * @default - BundlingFileCopyVariant.BIND_MOUNT + * @default - BundlingFileAccess.BIND_MOUNT */ - readonly assetStagingType?: BundlingFileCopyVariant; + readonly assetStagingType?: BundlingFileAccess; } /** @@ -115,7 +115,7 @@ export class Bundling implements cdk.BundlingOptions { public readonly user?: string; public readonly securityOpt?: string; public readonly network?: string; - public readonly assetStagingType?: cdk.BundlingFileCopyVariant; + public readonly assetStagingType?: cdk.BundlingFileAccess; private readonly relativeEntryPath: string; diff --git a/packages/@aws-cdk/aws-lambda-go/lib/types.ts b/packages/@aws-cdk/aws-lambda-go/lib/types.ts index 268c7dff17a0c..67a79f0b2543e 100644 --- a/packages/@aws-cdk/aws-lambda-go/lib/types.ts +++ b/packages/@aws-cdk/aws-lambda-go/lib/types.ts @@ -1,4 +1,4 @@ -import { AssetHashType, BundlingFileCopyVariant, DockerImage, DockerRunOptions } from '@aws-cdk/core'; +import { AssetHashType, BundlingFileAccess, DockerImage, DockerRunOptions } from '@aws-cdk/core'; /** * Bundling options @@ -114,9 +114,9 @@ export interface BundlingOptions extends DockerRunOptions { /** * Which option to use to copy the source files to the docker container and output files back - * @default - BundlingFileCopyVariant.BIND_MOUNT + * @default - BundlingFileAccess.BIND_MOUNT */ - readonly assetStagingType?: BundlingFileCopyVariant; + readonly assetStagingType?: BundlingFileAccess; } /** diff --git a/packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts b/packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts index e6088e78b1d09..8a396546d84da 100644 --- a/packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts +++ b/packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts @@ -2,7 +2,7 @@ import * as child_process from 'child_process'; import * as os from 'os'; import * as path from 'path'; import { Architecture, Code, Runtime } from '@aws-cdk/aws-lambda'; -import { AssetHashType, BundlingFileCopyVariant, DockerImage } from '@aws-cdk/core'; +import { AssetHashType, BundlingFileAccess, DockerImage } from '@aws-cdk/core'; import { Bundling } from '../lib/bundling'; import * as util from '../lib/util'; @@ -469,13 +469,13 @@ test('Custom bundling file copy variant', () => { runtime: Runtime.GO_1_X, architecture: Architecture.X86_64, forcedDockerBundling: true, - assetStagingType: BundlingFileCopyVariant.VOLUME_COPY, + assetStagingType: BundlingFileAccess.VOLUME_COPY, }); expect(Code.fromAsset).toHaveBeenCalledWith('/project', { assetHashType: AssetHashType.OUTPUT, bundling: expect.objectContaining({ - assetStagingType: BundlingFileCopyVariant.VOLUME_COPY, + assetStagingType: BundlingFileAccess.VOLUME_COPY, }), }); }); diff --git a/packages/@aws-cdk/aws-lambda-nodejs/README.md b/packages/@aws-cdk/aws-lambda-nodejs/README.md index 8fb5d3a2a68c8..ad4edbbc6100b 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/README.md +++ b/packages/@aws-cdk/aws-lambda-nodejs/README.md @@ -344,10 +344,10 @@ By default the input and output of Docker based bundling is handled via bind mou In situtations where this does not work, like Docker-in-Docker setups or when using a remote Docker socket, you can configure an alternative, but slower, variant that also works in these situations. ```ts -import { BundlingFileCopyVariant } from '@aws-cdk/core'; +import { BundlingFileAccess } from '@aws-cdk/core'; new nodejs.NodejsFunction(this, 'my-handler', { bundling: { - assetStagingType: BundlingFileCopyVariant.VOLUME_COPY, + assetStagingType: BundlingFileAccess.VOLUME_COPY, }, }); ``` diff --git a/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts b/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts index 80aaacbda2ce7..295a1d47ba41f 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts +++ b/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts @@ -6,7 +6,7 @@ import { PackageInstallation } from './package-installation'; import { LockFile, PackageManager } from './package-manager'; import { BundlingOptions, OutputFormat, SourceMapMode } from './types'; import { exec, extractDependencies, findUp, getTsconfigCompilerOptions } from './util'; -import { BundlingFileCopyVariant } from '@aws-cdk/core'; +import { BundlingFileAccess } from '@aws-cdk/core'; const ESBUILD_MAJOR_VERSION = '0'; @@ -46,9 +46,9 @@ export interface BundlingProps extends BundlingOptions { /** * Which option to use to copy the source files to the docker container and output files back - * @default - BundlingFileCopyVariant.BIND_MOUNT + * @default - BundlingFileAccess.BIND_MOUNT */ - readonly assetStagingType?: BundlingFileCopyVariant; + readonly assetStagingType?: BundlingFileAccess; } @@ -90,7 +90,7 @@ export class Bundling implements cdk.BundlingOptions { public readonly securityOpt?: string; public readonly network?: string; public readonly local?: cdk.ILocalBundling; - public readonly assetStagingType?: cdk.BundlingFileCopyVariant; + public readonly assetStagingType?: cdk.BundlingFileAccess; private readonly projectRoot: string; private readonly relativeEntryPath: string; diff --git a/packages/@aws-cdk/aws-lambda-nodejs/lib/types.ts b/packages/@aws-cdk/aws-lambda-nodejs/lib/types.ts index e26487d8b7c0f..8c0f014e3a78f 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/lib/types.ts +++ b/packages/@aws-cdk/aws-lambda-nodejs/lib/types.ts @@ -1,4 +1,4 @@ -import { BundlingFileCopyVariant, DockerImage, DockerRunOptions } from '@aws-cdk/core'; +import { BundlingFileAccess, DockerImage, DockerRunOptions } from '@aws-cdk/core'; /** * Bundling options @@ -304,9 +304,9 @@ export interface BundlingOptions extends DockerRunOptions { /** * Which option to use to copy the source files to the docker container and output files back - * @default - BundlingFileCopyVariant.BIND_MOUNT + * @default - BundlingFileAccess.BIND_MOUNT */ - readonly assetStagingType?: BundlingFileCopyVariant; + readonly assetStagingType?: BundlingFileAccess; } /** diff --git a/packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts b/packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts index a78e533b3a2f4..f7b458da1680c 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts +++ b/packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts @@ -2,7 +2,7 @@ import * as child_process from 'child_process'; import * as os from 'os'; import * as path from 'path'; import { Architecture, Code, Runtime, RuntimeFamily } from '@aws-cdk/aws-lambda'; -import { AssetHashType, BundlingFileCopyVariant, DockerImage } from '@aws-cdk/core'; +import { AssetHashType, BundlingFileAccess, DockerImage } from '@aws-cdk/core'; import { version as delayVersion } from 'delay/package.json'; import { Bundling } from '../lib/bundling'; import { PackageInstallation } from '../lib/package-installation'; @@ -835,13 +835,13 @@ test('Custom bundling file copy variant', () => { runtime: Runtime.NODEJS_14_X, architecture: Architecture.X86_64, forceDockerBundling: true, - assetStagingType: BundlingFileCopyVariant.VOLUME_COPY, + assetStagingType: BundlingFileAccess.VOLUME_COPY, }); expect(Code.fromAsset).toHaveBeenCalledWith('/project', { assetHashType: AssetHashType.OUTPUT, bundling: expect.objectContaining({ - assetStagingType: BundlingFileCopyVariant.VOLUME_COPY, + assetStagingType: BundlingFileAccess.VOLUME_COPY, }), }); }); diff --git a/packages/@aws-cdk/aws-lambda-python/README.md b/packages/@aws-cdk/aws-lambda-python/README.md index bff11d611f34d..c7fe25d431052 100644 --- a/packages/@aws-cdk/aws-lambda-python/README.md +++ b/packages/@aws-cdk/aws-lambda-python/README.md @@ -259,14 +259,14 @@ By default the input and output of Docker based bundling is handled via bind mou In situtations where this does not work, like Docker-in-Docker setups or when using a remote Docker socket, you can configure an alternative, but slower, variant that also works in these situations. ```ts - import { BundlingFileCopyVariant } from '@aws-cdk/core'; + import { BundlingFileAccess } from '@aws-cdk/core'; const entry = '/path/to/function'; new python.PythonFunction(this, 'function', { entry, runtime: Runtime.PYTHON_3_8, bundling: { - assetStagingType: BundlingFileCopyVariant.VOLUME_COPY, + assetStagingType: BundlingFileAccess.VOLUME_COPY, }, }); ``` diff --git a/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts b/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts index 9b9c09bdf3e05..d460f3654a8fa 100644 --- a/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts +++ b/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts @@ -1,6 +1,6 @@ import * as path from 'path'; import { Architecture, AssetCode, Code, Runtime } from '@aws-cdk/aws-lambda'; -import { AssetStaging, BundlingFileCopyVariant, BundlingOptions as CdkBundlingOptions, DockerImage, DockerVolume } from '@aws-cdk/core'; +import { AssetStaging, BundlingFileAccess, BundlingOptions as CdkBundlingOptions, DockerImage, DockerVolume } from '@aws-cdk/core'; import { Packaging, DependenciesFile } from './packaging'; import { BundlingOptions, ICommandHooks } from './types'; @@ -44,9 +44,9 @@ export interface BundlingProps extends BundlingOptions { /** * Which option to use to copy the source files to the docker container and output files back - * @default - BundlingFileCopyVariant.BIND_MOUNT + * @default - BundlingFileAccess.BIND_MOUNT */ - assetStagingType?: BundlingFileCopyVariant + assetStagingType?: BundlingFileAccess } /** @@ -72,7 +72,7 @@ export class Bundling implements CdkBundlingOptions { public readonly user?: string; public readonly securityOpt?: string; public readonly network?: string; - public readonly assetStagingType?: BundlingFileCopyVariant; + public readonly assetStagingType?: BundlingFileAccess; constructor(props: BundlingProps) { const { diff --git a/packages/@aws-cdk/aws-lambda-python/lib/types.ts b/packages/@aws-cdk/aws-lambda-python/lib/types.ts index 39add7fd7b4b5..c2adc2da5fc0c 100644 --- a/packages/@aws-cdk/aws-lambda-python/lib/types.ts +++ b/packages/@aws-cdk/aws-lambda-python/lib/types.ts @@ -1,4 +1,4 @@ -import { AssetHashType, BundlingFileCopyVariant, DockerImage, DockerRunOptions } from '@aws-cdk/core'; +import { AssetHashType, BundlingFileAccess, DockerImage, DockerRunOptions } from '@aws-cdk/core'; /** @@ -89,9 +89,9 @@ export interface BundlingOptions extends DockerRunOptions { /** * Which option to use to copy the source files to the docker container and output files back - * @default - BundlingFileCopyVariant.BIND_MOUNT + * @default - BundlingFileAccess.BIND_MOUNT */ - readonly assetStagingType?: BundlingFileCopyVariant; + readonly assetStagingType?: BundlingFileAccess; } /** diff --git a/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts b/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts index 79ee41739fb87..268e4223afcd1 100644 --- a/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts +++ b/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts @@ -1,7 +1,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { Architecture, Code, Runtime } from '@aws-cdk/aws-lambda'; -import { BundlingFileCopyVariant, DockerImage } from '@aws-cdk/core'; +import { BundlingFileAccess, DockerImage } from '@aws-cdk/core'; import { Bundling } from '../lib/bundling'; jest.spyOn(Code, 'fromAsset'); @@ -379,13 +379,13 @@ test('Bundling with docker copy variant', () => { Bundling.bundle({ entry: entry, runtime: Runtime.PYTHON_3_7, - assetStagingType: BundlingFileCopyVariant.VOLUME_COPY, + assetStagingType: BundlingFileAccess.VOLUME_COPY, }); expect(Code.fromAsset).toHaveBeenCalledWith(entry, expect.objectContaining({ bundling: expect.objectContaining({ - assetStagingType: BundlingFileCopyVariant.VOLUME_COPY, + assetStagingType: BundlingFileAccess.VOLUME_COPY, }), })); }); diff --git a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.ts b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.ts index 2aa4e91b8daf4..aad151e3b81da 100644 --- a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.ts +++ b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.ts @@ -3,7 +3,7 @@ /// !cdk-integ pragma:disable-update-workflow import * as path from 'path'; import { Runtime } from '@aws-cdk/aws-lambda'; -import { App, CfnOutput, Stack, StackProps, BundlingFileCopyVariant } from '@aws-cdk/core'; +import { App, CfnOutput, Stack, StackProps, BundlingFileAccess } from '@aws-cdk/core'; import { IntegTest, ExpectedResult } from '@aws-cdk/integ-tests'; import { Construct } from 'constructs'; import * as lambda from '../lib'; @@ -22,7 +22,7 @@ class TestStack extends Stack { entry: path.join(__dirname, 'lambda-handler-dockercopy'), runtime: Runtime.PYTHON_3_9, bundling: { - assetStagingType: BundlingFileCopyVariant.VOLUME_COPY, + assetStagingType: BundlingFileAccess.VOLUME_COPY, }, }); this.functionName = fn.functionName; diff --git a/packages/@aws-cdk/core/lib/asset-staging.ts b/packages/@aws-cdk/core/lib/asset-staging.ts index d9ffdfd4f2a7c..58144f7ebd17e 100644 --- a/packages/@aws-cdk/core/lib/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/asset-staging.ts @@ -4,11 +4,11 @@ import * as cxapi from '@aws-cdk/cx-api'; import { Construct } from 'constructs'; import * as fs from 'fs-extra'; import { AssetHashType, AssetOptions, FileAssetPackaging } from './assets'; -import { BundlingFileCopyVariant, BundlingOptions, BundlingOutput } from './bundling'; +import { BundlingFileAccess, BundlingOptions, BundlingOutput } from './bundling'; import { FileSystem, FingerprintOptions } from './fs'; import { clearLargeFileFingerprintCache } from './fs/fingerprint'; import { Names } from './names'; -import { AssetStagingVolumeCopy, AssetStagingBindMount } from './private/asset-staging'; +import { AssetBundlingVolumeCopy, AssetBundlingBindMount } from './private/asset-staging'; import { Cache } from './private/cache'; import { Stack } from './stack'; import { Stage } from './stage'; @@ -445,12 +445,12 @@ export class AssetStaging extends Construct { }; switch (options.assetStagingType) { - case BundlingFileCopyVariant.VOLUME_COPY: - new AssetStagingVolumeCopy(assetStagingOptions).run(); + case BundlingFileAccess.VOLUME_COPY: + new AssetBundlingVolumeCopy(assetStagingOptions).run(); break; - case BundlingFileCopyVariant.BIND_MOUNT: + case BundlingFileAccess.BIND_MOUNT: default: - new AssetStagingBindMount(assetStagingOptions).run(); + new AssetBundlingBindMount(assetStagingOptions).run(); break; } } diff --git a/packages/@aws-cdk/core/lib/bundling.ts b/packages/@aws-cdk/core/lib/bundling.ts index aaa40e786f9f9..ce497236c19e0 100644 --- a/packages/@aws-cdk/core/lib/bundling.ts +++ b/packages/@aws-cdk/core/lib/bundling.ts @@ -112,9 +112,9 @@ export interface BundlingOptions { /** * The Which option to use to copy the source files to the docker container and output files back - * @default - BundlingFileCopyVariant.BIND_MOUNT + * @default - BundlingFileAccess.BIND_MOUNT */ - readonly assetStagingType?: BundlingFileCopyVariant; + readonly assetStagingType?: BundlingFileAccess; } /** @@ -161,7 +161,7 @@ export interface ILocalBundling { /** * The type of file copy that should be used for bundling */ -export enum BundlingFileCopyVariant { +export enum BundlingFileAccess { /** * Creates temporary volumes and containers to copy files from the host to the bundling container and back. * This is slower, but works also in more complex situations with remote or shared docker sockets. @@ -173,10 +173,8 @@ export enum BundlingFileCopyVariant { * This is faster and simpler, but less portable than the other option. */ BIND_MOUNT = 'BIND_MOUNT', - } - /** * A Docker image used for asset bundling * diff --git a/packages/@aws-cdk/core/lib/private/asset-staging.ts b/packages/@aws-cdk/core/lib/private/asset-staging.ts index 9d2ac8a33f96f..27cfbfc604152 100644 --- a/packages/@aws-cdk/core/lib/private/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/private/asset-staging.ts @@ -7,7 +7,7 @@ import { BundlingOptions } from '../bundling'; /** * Options for Docker based bundling of assets */ -interface AssetStagingCopyOptions extends BundlingOptions { +interface AssetBundlingOptions extends BundlingOptions { /** * Path where the source files are located */ @@ -18,9 +18,9 @@ interface AssetStagingCopyOptions extends BundlingOptions { readonly bundleDir: string; } -class AssetStagingBase { - protected options: AssetStagingCopyOptions; - constructor(options: AssetStagingCopyOptions) { +abstract class AssetBundlingBase { + protected options: AssetBundlingOptions; + constructor(options: AssetBundlingOptions) { this.options = options; } /** @@ -45,7 +45,7 @@ class AssetStagingBase { /** * Bundles files with bind mount as copy method */ -export class AssetStagingBindMount extends AssetStagingBase { +export class AssetBundlingBindMount extends AssetBundlingBase { /** * Bundle files with bind mount as copy method */ @@ -77,7 +77,7 @@ export class AssetStagingBindMount extends AssetStagingBase { /** * Provides a helper container for copying bundling related files to specific input and output volumes */ -export class AssetStagingVolumeCopy extends AssetStagingBase { +export class AssetBundlingVolumeCopy extends AssetBundlingBase { /** * Name of the Docker volume that is used for the asset input */ @@ -91,7 +91,7 @@ export class AssetStagingVolumeCopy extends AssetStagingBase { */ public copyContainerName: string; - constructor(options: AssetStagingCopyOptions) { + constructor(options: AssetBundlingOptions) { super(options); const copySuffix = crypto.randomBytes(12).toString('hex'); this.inputVolumeName = `assetInput${copySuffix}`; diff --git a/packages/@aws-cdk/core/test/private/asset-staging.test.ts b/packages/@aws-cdk/core/test/private/asset-staging.test.ts index 825b78bd59e35..f45271297677e 100644 --- a/packages/@aws-cdk/core/test/private/asset-staging.test.ts +++ b/packages/@aws-cdk/core/test/private/asset-staging.test.ts @@ -1,7 +1,7 @@ import * as child_process from 'child_process'; import * as sinon from 'sinon'; import { AssetStaging, DockerImage } from '../../lib'; -import { AssetStagingBindMount, AssetStagingVolumeCopy } from '../../lib/private/asset-staging'; +import { AssetBundlingBindMount, AssetBundlingVolumeCopy } from '../../lib/private/asset-staging'; describe('bundling', () => { afterEach(() => { @@ -9,7 +9,7 @@ describe('bundling', () => { }); - test('AssetStagingVolumeCopy bundles with volume copy ', () => { + test('AssetBundlingVolumeCopy bundles with volume copy ', () => { // GIVEN sinon.stub(process, 'platform').value('darwin'); const spawnSyncStub = sinon.stub(child_process, 'spawnSync').returns({ @@ -26,7 +26,7 @@ describe('bundling', () => { image: DockerImage.fromRegistry('alpine'), user: '1000', }; - const helper = new AssetStagingVolumeCopy(options); + const helper = new AssetBundlingVolumeCopy(options); helper.run(); // volume Creation @@ -83,7 +83,7 @@ describe('bundling', () => { }); - test('AssetStagingBindMount bundles with bind mount ', () => { + test('AssetBundlingBindMount bundles with bind mount ', () => { // GIVEN sinon.stub(process, 'platform').value('darwin'); const spawnSyncStub = sinon.stub(child_process, 'spawnSync').returns({ @@ -100,7 +100,7 @@ describe('bundling', () => { image: DockerImage.fromRegistry('alpine'), user: '1000', }; - const helper = new AssetStagingBindMount(options); + const helper = new AssetBundlingBindMount(options); helper.run(); // actual docker run with bind mount is called diff --git a/packages/@aws-cdk/core/test/staging.test.ts b/packages/@aws-cdk/core/test/staging.test.ts index 8db07a2f94579..603012af8a7b5 100644 --- a/packages/@aws-cdk/core/test/staging.test.ts +++ b/packages/@aws-cdk/core/test/staging.test.ts @@ -6,7 +6,7 @@ import { FileAssetPackaging } from '@aws-cdk/cloud-assembly-schema'; import * as cxapi from '@aws-cdk/cx-api'; import * as fs from 'fs-extra'; import * as sinon from 'sinon'; -import { App, AssetHashType, AssetStaging, DockerImage, BundlingOptions, BundlingOutput, FileSystem, Stack, Stage, BundlingFileCopyVariant } from '../lib'; +import { App, AssetHashType, AssetStaging, DockerImage, BundlingOptions, BundlingOutput, FileSystem, Stack, Stage, BundlingFileAccess } from '../lib'; const STUB_INPUT_FILE = '/tmp/docker-stub.input'; const STUB_INPUT_CONCAT_FILE = '/tmp/docker-stub.input.concat'; @@ -1296,7 +1296,7 @@ describe('staging with docker cp', () => { bundling: { image: DockerImage.fromRegistry('alpine'), command: [DockerStubCommand.VOLUME_SINGLE_ARCHIVE], - assetStagingType: BundlingFileCopyVariant.VOLUME_COPY, + assetStagingType: BundlingFileAccess.VOLUME_COPY, }, }); From 974762a169206d1dd3725a8f5edc23b8fcd3f990 Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Wed, 25 Jan 2023 16:57:43 +0100 Subject: [PATCH 30/34] Update packages/@aws-cdk/core/lib/bundling.ts Co-authored-by: Momo Kornher --- packages/@aws-cdk/core/lib/bundling.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/core/lib/bundling.ts b/packages/@aws-cdk/core/lib/bundling.ts index ce497236c19e0..dee4a2e697097 100644 --- a/packages/@aws-cdk/core/lib/bundling.ts +++ b/packages/@aws-cdk/core/lib/bundling.ts @@ -111,7 +111,7 @@ export interface BundlingOptions { readonly network?: string; /** - * The Which option to use to copy the source files to the docker container and output files back + * The access mechanism used to make source files available to the bundling container and to return the bundling output back to the host. * @default - BundlingFileAccess.BIND_MOUNT */ readonly assetStagingType?: BundlingFileAccess; From 2890d541fa5bf3c2b327e2934fb5a20e41f333ae Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Wed, 25 Jan 2023 16:57:54 +0100 Subject: [PATCH 31/34] Update packages/@aws-cdk/core/lib/bundling.ts Co-authored-by: Momo Kornher --- packages/@aws-cdk/core/lib/bundling.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/core/lib/bundling.ts b/packages/@aws-cdk/core/lib/bundling.ts index dee4a2e697097..3fc8dc9b02f8d 100644 --- a/packages/@aws-cdk/core/lib/bundling.ts +++ b/packages/@aws-cdk/core/lib/bundling.ts @@ -159,7 +159,7 @@ export interface ILocalBundling { } /** - * The type of file copy that should be used for bundling + * The access mechanism used to make source files available to the bundling container and to return the bundling output back to the host */ export enum BundlingFileAccess { /** From 325fdbb9bf64eddffbd4edda7166227b61e65a52 Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Wed, 25 Jan 2023 16:58:07 +0100 Subject: [PATCH 32/34] Update packages/@aws-cdk/core/lib/bundling.ts Co-authored-by: Momo Kornher --- packages/@aws-cdk/core/lib/bundling.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/@aws-cdk/core/lib/bundling.ts b/packages/@aws-cdk/core/lib/bundling.ts index 3fc8dc9b02f8d..e894795eb5b5e 100644 --- a/packages/@aws-cdk/core/lib/bundling.ts +++ b/packages/@aws-cdk/core/lib/bundling.ts @@ -170,7 +170,7 @@ export enum BundlingFileAccess { /** * The source and output folders will be mounted as bind mount from the host system - * This is faster and simpler, but less portable than the other option. + * This is faster and simpler, but less portable than `VOLUME_COPY`. */ BIND_MOUNT = 'BIND_MOUNT', } From 6590681f39ef8722c29564dcbfe64743340fca49 Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Thu, 26 Jan 2023 15:26:23 +0100 Subject: [PATCH 33/34] remaining name fixes --- packages/@aws-cdk/aws-lambda-go/README.md | 2 +- packages/@aws-cdk/aws-lambda-go/lib/bundling.ts | 8 ++++---- packages/@aws-cdk/aws-lambda-go/lib/types.ts | 2 +- packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts | 4 ++-- packages/@aws-cdk/aws-lambda-nodejs/README.md | 2 +- packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts | 6 +++--- packages/@aws-cdk/aws-lambda-nodejs/lib/types.ts | 2 +- packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts | 4 ++-- packages/@aws-cdk/aws-lambda-python/README.md | 2 +- packages/@aws-cdk/aws-lambda-python/lib/bundling.ts | 6 +++--- packages/@aws-cdk/aws-lambda-python/lib/types.ts | 2 +- packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts | 4 ++-- .../aws-lambda-python/test/integ.function.dockercopy.ts | 2 +- packages/@aws-cdk/core/lib/asset-staging.ts | 2 +- packages/@aws-cdk/core/lib/bundling.ts | 2 +- packages/@aws-cdk/core/test/staging.test.ts | 8 ++++---- 16 files changed, 29 insertions(+), 29 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-go/README.md b/packages/@aws-cdk/aws-lambda-go/README.md index a11da09adcb60..2165f5a177201 100644 --- a/packages/@aws-cdk/aws-lambda-go/README.md +++ b/packages/@aws-cdk/aws-lambda-go/README.md @@ -295,7 +295,7 @@ import { BundlingFileAccess } from '@aws-cdk/core'; new go.GoFunction(this, 'GoFunction', { entry: 'app/cmd/api', bundling: { - assetStagingType: BundlingFileAccess.VOLUME_COPY, + bundlingFileAccess: BundlingFileAccess.VOLUME_COPY, }, }); ``` diff --git a/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts b/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts index 3a7f21e49928f..dd609c8366bdd 100644 --- a/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts +++ b/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts @@ -66,7 +66,7 @@ export interface BundlingProps extends BundlingOptions { * Which option to use to copy the source files to the docker container and output files back * @default - BundlingFileAccess.BIND_MOUNT */ - readonly assetStagingType?: BundlingFileAccess; + readonly bundlingFileAccess?: BundlingFileAccess; } /** @@ -92,7 +92,7 @@ export class Bundling implements cdk.BundlingOptions { user: bundling.user, securityOpt: bundling.securityOpt, network: bundling.network, - assetStagingType: bundling.assetStagingType, + bundlingFileAccess: bundling.bundlingFileAccess, }, }); } @@ -115,7 +115,7 @@ export class Bundling implements cdk.BundlingOptions { public readonly user?: string; public readonly securityOpt?: string; public readonly network?: string; - public readonly assetStagingType?: cdk.BundlingFileAccess; + public readonly bundlingFileAccess?: cdk.BundlingFileAccess; private readonly relativeEntryPath: string; @@ -163,7 +163,7 @@ export class Bundling implements cdk.BundlingOptions { this.user = props.user; this.securityOpt = props.securityOpt; this.network = props.network; - this.assetStagingType = props.assetStagingType; + this.bundlingFileAccess = props.bundlingFileAccess; // Local bundling if (!props.forcedDockerBundling) { // only if Docker is not forced diff --git a/packages/@aws-cdk/aws-lambda-go/lib/types.ts b/packages/@aws-cdk/aws-lambda-go/lib/types.ts index 67a79f0b2543e..addadbe09a01b 100644 --- a/packages/@aws-cdk/aws-lambda-go/lib/types.ts +++ b/packages/@aws-cdk/aws-lambda-go/lib/types.ts @@ -116,7 +116,7 @@ export interface BundlingOptions extends DockerRunOptions { * Which option to use to copy the source files to the docker container and output files back * @default - BundlingFileAccess.BIND_MOUNT */ - readonly assetStagingType?: BundlingFileAccess; + readonly bundlingFileAccess?: BundlingFileAccess; } /** diff --git a/packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts b/packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts index 8a396546d84da..af0419d879699 100644 --- a/packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts +++ b/packages/@aws-cdk/aws-lambda-go/test/bundling.test.ts @@ -469,13 +469,13 @@ test('Custom bundling file copy variant', () => { runtime: Runtime.GO_1_X, architecture: Architecture.X86_64, forcedDockerBundling: true, - assetStagingType: BundlingFileAccess.VOLUME_COPY, + bundlingFileAccess: BundlingFileAccess.VOLUME_COPY, }); expect(Code.fromAsset).toHaveBeenCalledWith('/project', { assetHashType: AssetHashType.OUTPUT, bundling: expect.objectContaining({ - assetStagingType: BundlingFileAccess.VOLUME_COPY, + bundlingFileAccess: BundlingFileAccess.VOLUME_COPY, }), }); }); diff --git a/packages/@aws-cdk/aws-lambda-nodejs/README.md b/packages/@aws-cdk/aws-lambda-nodejs/README.md index ad4edbbc6100b..54729afd0ee5c 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/README.md +++ b/packages/@aws-cdk/aws-lambda-nodejs/README.md @@ -347,7 +347,7 @@ In situtations where this does not work, like Docker-in-Docker setups or when us import { BundlingFileAccess } from '@aws-cdk/core'; new nodejs.NodejsFunction(this, 'my-handler', { bundling: { - assetStagingType: BundlingFileAccess.VOLUME_COPY, + bundlingFileAccess: BundlingFileAccess.VOLUME_COPY, }, }); ``` diff --git a/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts b/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts index 295a1d47ba41f..0a01af8b3855e 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts +++ b/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts @@ -48,7 +48,7 @@ export interface BundlingProps extends BundlingOptions { * Which option to use to copy the source files to the docker container and output files back * @default - BundlingFileAccess.BIND_MOUNT */ - readonly assetStagingType?: BundlingFileAccess; + readonly bundlingFileAccess?: BundlingFileAccess; } @@ -90,7 +90,7 @@ export class Bundling implements cdk.BundlingOptions { public readonly securityOpt?: string; public readonly network?: string; public readonly local?: cdk.ILocalBundling; - public readonly assetStagingType?: cdk.BundlingFileAccess; + public readonly bundlingFileAccess?: cdk.BundlingFileAccess; private readonly projectRoot: string; private readonly relativeEntryPath: string; @@ -162,7 +162,7 @@ export class Bundling implements cdk.BundlingOptions { this.user = props.user; this.securityOpt = props.securityOpt; this.network = props.network; - this.assetStagingType = props.assetStagingType; + this.bundlingFileAccess = props.bundlingFileAccess; // Local bundling if (!props.forceDockerBundling) { // only if Docker is not forced diff --git a/packages/@aws-cdk/aws-lambda-nodejs/lib/types.ts b/packages/@aws-cdk/aws-lambda-nodejs/lib/types.ts index 8c0f014e3a78f..1c123166bfac1 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/lib/types.ts +++ b/packages/@aws-cdk/aws-lambda-nodejs/lib/types.ts @@ -306,7 +306,7 @@ export interface BundlingOptions extends DockerRunOptions { * Which option to use to copy the source files to the docker container and output files back * @default - BundlingFileAccess.BIND_MOUNT */ - readonly assetStagingType?: BundlingFileAccess; + readonly bundlingFileAccess?: BundlingFileAccess; } /** diff --git a/packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts b/packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts index f7b458da1680c..5d5943d036195 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts +++ b/packages/@aws-cdk/aws-lambda-nodejs/test/bundling.test.ts @@ -835,13 +835,13 @@ test('Custom bundling file copy variant', () => { runtime: Runtime.NODEJS_14_X, architecture: Architecture.X86_64, forceDockerBundling: true, - assetStagingType: BundlingFileAccess.VOLUME_COPY, + bundlingFileAccess: BundlingFileAccess.VOLUME_COPY, }); expect(Code.fromAsset).toHaveBeenCalledWith('/project', { assetHashType: AssetHashType.OUTPUT, bundling: expect.objectContaining({ - assetStagingType: BundlingFileAccess.VOLUME_COPY, + bundlingFileAccess: BundlingFileAccess.VOLUME_COPY, }), }); }); diff --git a/packages/@aws-cdk/aws-lambda-python/README.md b/packages/@aws-cdk/aws-lambda-python/README.md index c7fe25d431052..b7d159cc3b346 100644 --- a/packages/@aws-cdk/aws-lambda-python/README.md +++ b/packages/@aws-cdk/aws-lambda-python/README.md @@ -266,7 +266,7 @@ new python.PythonFunction(this, 'function', { entry, runtime: Runtime.PYTHON_3_8, bundling: { - assetStagingType: BundlingFileAccess.VOLUME_COPY, + bundlingFileAccess: BundlingFileAccess.VOLUME_COPY, }, }); ``` diff --git a/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts b/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts index d460f3654a8fa..fad11266bb0cb 100644 --- a/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts +++ b/packages/@aws-cdk/aws-lambda-python/lib/bundling.ts @@ -46,7 +46,7 @@ export interface BundlingProps extends BundlingOptions { * Which option to use to copy the source files to the docker container and output files back * @default - BundlingFileAccess.BIND_MOUNT */ - assetStagingType?: BundlingFileAccess + bundlingFileAccess?: BundlingFileAccess } /** @@ -72,7 +72,7 @@ export class Bundling implements CdkBundlingOptions { public readonly user?: string; public readonly securityOpt?: string; public readonly network?: string; - public readonly assetStagingType?: BundlingFileAccess; + public readonly bundlingFileAccess?: BundlingFileAccess; constructor(props: BundlingProps) { const { @@ -111,7 +111,7 @@ export class Bundling implements CdkBundlingOptions { this.user = props.user; this.securityOpt = props.securityOpt; this.network = props.network; - this.assetStagingType = props.assetStagingType; + this.bundlingFileAccess = props.bundlingFileAccess; } private createBundlingCommand(options: BundlingCommandOptions): string[] { diff --git a/packages/@aws-cdk/aws-lambda-python/lib/types.ts b/packages/@aws-cdk/aws-lambda-python/lib/types.ts index c2adc2da5fc0c..e0d328e68c858 100644 --- a/packages/@aws-cdk/aws-lambda-python/lib/types.ts +++ b/packages/@aws-cdk/aws-lambda-python/lib/types.ts @@ -91,7 +91,7 @@ export interface BundlingOptions extends DockerRunOptions { * Which option to use to copy the source files to the docker container and output files back * @default - BundlingFileAccess.BIND_MOUNT */ - readonly assetStagingType?: BundlingFileAccess; + readonly bundlingFileAccess?: BundlingFileAccess; } /** diff --git a/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts b/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts index 268e4223afcd1..b51e8bb046acc 100644 --- a/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts +++ b/packages/@aws-cdk/aws-lambda-python/test/bundling.test.ts @@ -379,13 +379,13 @@ test('Bundling with docker copy variant', () => { Bundling.bundle({ entry: entry, runtime: Runtime.PYTHON_3_7, - assetStagingType: BundlingFileAccess.VOLUME_COPY, + bundlingFileAccess: BundlingFileAccess.VOLUME_COPY, }); expect(Code.fromAsset).toHaveBeenCalledWith(entry, expect.objectContaining({ bundling: expect.objectContaining({ - assetStagingType: BundlingFileAccess.VOLUME_COPY, + bundlingFileAccess: BundlingFileAccess.VOLUME_COPY, }), })); }); diff --git a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.ts b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.ts index aad151e3b81da..6cf03198fb624 100644 --- a/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.ts +++ b/packages/@aws-cdk/aws-lambda-python/test/integ.function.dockercopy.ts @@ -22,7 +22,7 @@ class TestStack extends Stack { entry: path.join(__dirname, 'lambda-handler-dockercopy'), runtime: Runtime.PYTHON_3_9, bundling: { - assetStagingType: BundlingFileAccess.VOLUME_COPY, + bundlingFileAccess: BundlingFileAccess.VOLUME_COPY, }, }); this.functionName = fn.functionName; diff --git a/packages/@aws-cdk/core/lib/asset-staging.ts b/packages/@aws-cdk/core/lib/asset-staging.ts index 58144f7ebd17e..aae0b1641f077 100644 --- a/packages/@aws-cdk/core/lib/asset-staging.ts +++ b/packages/@aws-cdk/core/lib/asset-staging.ts @@ -444,7 +444,7 @@ export class AssetStaging extends Construct { ...options, }; - switch (options.assetStagingType) { + switch (options.bundlingFileAccess) { case BundlingFileAccess.VOLUME_COPY: new AssetBundlingVolumeCopy(assetStagingOptions).run(); break; diff --git a/packages/@aws-cdk/core/lib/bundling.ts b/packages/@aws-cdk/core/lib/bundling.ts index e894795eb5b5e..d88a84043a652 100644 --- a/packages/@aws-cdk/core/lib/bundling.ts +++ b/packages/@aws-cdk/core/lib/bundling.ts @@ -114,7 +114,7 @@ export interface BundlingOptions { * The access mechanism used to make source files available to the bundling container and to return the bundling output back to the host. * @default - BundlingFileAccess.BIND_MOUNT */ - readonly assetStagingType?: BundlingFileAccess; + readonly bundlingFileAccess?: BundlingFileAccess; } /** diff --git a/packages/@aws-cdk/core/test/staging.test.ts b/packages/@aws-cdk/core/test/staging.test.ts index 603012af8a7b5..a13f23c495613 100644 --- a/packages/@aws-cdk/core/test/staging.test.ts +++ b/packages/@aws-cdk/core/test/staging.test.ts @@ -1296,21 +1296,21 @@ describe('staging with docker cp', () => { bundling: { image: DockerImage.fromRegistry('alpine'), command: [DockerStubCommand.VOLUME_SINGLE_ARCHIVE], - assetStagingType: BundlingFileAccess.VOLUME_COPY, + bundlingFileAccess: BundlingFileAccess.VOLUME_COPY, }, }); // THEN const assembly = app.synth(); expect(fs.readdirSync(assembly.directory)).toEqual([ - 'asset.812ead30bdacbf9ae61ac91c52e460755a9b072440e703620ab7f8cd09eaa78f', // this is the bundle dir - 'asset.812ead30bdacbf9ae61ac91c52e460755a9b072440e703620ab7f8cd09eaa78f.zip', + 'asset.0ec371a2022d29dfd83f5df104e0f01b34233a4e3e839c3c4ec62008f0b9a0e8', // this is the bundle dir + 'asset.0ec371a2022d29dfd83f5df104e0f01b34233a4e3e839c3c4ec62008f0b9a0e8.zip', 'cdk.out', 'manifest.json', 'stack.template.json', 'tree.json', ]); - expect(fs.readdirSync(path.join(assembly.directory, 'asset.812ead30bdacbf9ae61ac91c52e460755a9b072440e703620ab7f8cd09eaa78f'))).toEqual([ + expect(fs.readdirSync(path.join(assembly.directory, 'asset.0ec371a2022d29dfd83f5df104e0f01b34233a4e3e839c3c4ec62008f0b9a0e8'))).toEqual([ 'test.zip', // bundle dir with "touched" bundled output file ]); expect(staging.packaging).toEqual(FileAssetPackaging.FILE); From 4ecaf47d1f47980a0b57f8180cc2dc63cd316486 Mon Sep 17 00:00:00 2001 From: Andreas Sieferlinger Date: Thu, 26 Jan 2023 15:51:13 +0100 Subject: [PATCH 34/34] cleanup --- packages/@aws-cdk/aws-lambda-go/README.md | 1 - packages/@aws-cdk/aws-lambda-go/lib/bundling.ts | 3 +-- packages/@aws-cdk/aws-lambda-go/rosetta/default.ts-fixture | 2 +- packages/@aws-cdk/aws-lambda-nodejs/README.md | 1 - packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts | 4 +--- .../@aws-cdk/aws-lambda-nodejs/rosetta/default.ts-fixture | 2 +- packages/@aws-cdk/aws-lambda-python/README.md | 7 +++---- .../@aws-cdk/aws-lambda-python/rosetta/default.ts-fixture | 2 +- packages/@aws-cdk/core/test/private/asset-staging.test.ts | 3 --- packages/@aws-cdk/core/test/staging.test.ts | 1 - 10 files changed, 8 insertions(+), 18 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda-go/README.md b/packages/@aws-cdk/aws-lambda-go/README.md index 2165f5a177201..530f3b56cf143 100644 --- a/packages/@aws-cdk/aws-lambda-go/README.md +++ b/packages/@aws-cdk/aws-lambda-go/README.md @@ -291,7 +291,6 @@ By default the input and output of Docker based bundling is handled via bind mou In situtations where this does not work, like Docker-in-Docker setups or when using a remote Docker socket, you can configure an alternative, but slower, variant that also works in these situations. ```ts -import { BundlingFileAccess } from '@aws-cdk/core'; new go.GoFunction(this, 'GoFunction', { entry: 'app/cmd/api', bundling: { diff --git a/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts b/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts index dd609c8366bdd..54fce951d488b 100644 --- a/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts +++ b/packages/@aws-cdk/aws-lambda-go/lib/bundling.ts @@ -2,7 +2,6 @@ import * as os from 'os'; import * as path from 'path'; import { Architecture, AssetCode, Code, Runtime } from '@aws-cdk/aws-lambda'; import * as cdk from '@aws-cdk/core'; -import { BundlingFileAccess } from '@aws-cdk/core'; import { BundlingOptions } from './types'; import { exec, findUp, getGoBuildVersion } from './util'; @@ -66,7 +65,7 @@ export interface BundlingProps extends BundlingOptions { * Which option to use to copy the source files to the docker container and output files back * @default - BundlingFileAccess.BIND_MOUNT */ - readonly bundlingFileAccess?: BundlingFileAccess; + readonly bundlingFileAccess?: cdk.BundlingFileAccess; } /** diff --git a/packages/@aws-cdk/aws-lambda-go/rosetta/default.ts-fixture b/packages/@aws-cdk/aws-lambda-go/rosetta/default.ts-fixture index 5daf6825a50ad..6f58572f1bcf4 100644 --- a/packages/@aws-cdk/aws-lambda-go/rosetta/default.ts-fixture +++ b/packages/@aws-cdk/aws-lambda-go/rosetta/default.ts-fixture @@ -1,6 +1,6 @@ // Fixture with packages imported, but nothing else import { Construct } from 'constructs'; -import { DockerImage, Stack } from '@aws-cdk/core'; +import { DockerImage, Stack, BundlingFileAccess } from '@aws-cdk/core'; import * as go from '@aws-cdk/aws-lambda-go'; class Fixture extends Stack { diff --git a/packages/@aws-cdk/aws-lambda-nodejs/README.md b/packages/@aws-cdk/aws-lambda-nodejs/README.md index 54729afd0ee5c..a895188db5206 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/README.md +++ b/packages/@aws-cdk/aws-lambda-nodejs/README.md @@ -344,7 +344,6 @@ By default the input and output of Docker based bundling is handled via bind mou In situtations where this does not work, like Docker-in-Docker setups or when using a remote Docker socket, you can configure an alternative, but slower, variant that also works in these situations. ```ts -import { BundlingFileAccess } from '@aws-cdk/core'; new nodejs.NodejsFunction(this, 'my-handler', { bundling: { bundlingFileAccess: BundlingFileAccess.VOLUME_COPY, diff --git a/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts b/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts index 0a01af8b3855e..9402079a1253a 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts +++ b/packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts @@ -6,7 +6,6 @@ import { PackageInstallation } from './package-installation'; import { LockFile, PackageManager } from './package-manager'; import { BundlingOptions, OutputFormat, SourceMapMode } from './types'; import { exec, extractDependencies, findUp, getTsconfigCompilerOptions } from './util'; -import { BundlingFileAccess } from '@aws-cdk/core'; const ESBUILD_MAJOR_VERSION = '0'; @@ -48,8 +47,7 @@ export interface BundlingProps extends BundlingOptions { * Which option to use to copy the source files to the docker container and output files back * @default - BundlingFileAccess.BIND_MOUNT */ - readonly bundlingFileAccess?: BundlingFileAccess; - + readonly bundlingFileAccess?: cdk.BundlingFileAccess; } /** diff --git a/packages/@aws-cdk/aws-lambda-nodejs/rosetta/default.ts-fixture b/packages/@aws-cdk/aws-lambda-nodejs/rosetta/default.ts-fixture index 0414866994604..0e9604aa39b08 100644 --- a/packages/@aws-cdk/aws-lambda-nodejs/rosetta/default.ts-fixture +++ b/packages/@aws-cdk/aws-lambda-nodejs/rosetta/default.ts-fixture @@ -1,6 +1,6 @@ // Fixture with packages imported, but nothing else import { Construct } from 'constructs'; -import { DockerImage, Stack } from '@aws-cdk/core'; +import { DockerImage, Stack, BundlingFileAccess } from '@aws-cdk/core'; import * as nodejs from '@aws-cdk/aws-lambda-nodejs'; class Fixture extends Stack { diff --git a/packages/@aws-cdk/aws-lambda-python/README.md b/packages/@aws-cdk/aws-lambda-python/README.md index b7d159cc3b346..f399d1559c81a 100644 --- a/packages/@aws-cdk/aws-lambda-python/README.md +++ b/packages/@aws-cdk/aws-lambda-python/README.md @@ -258,15 +258,14 @@ container for Docker bundling or on the host OS for local bundling. By default the input and output of Docker based bundling is handled via bind mounts. In situtations where this does not work, like Docker-in-Docker setups or when using a remote Docker socket, you can configure an alternative, but slower, variant that also works in these situations. - ```ts - import { BundlingFileAccess } from '@aws-cdk/core'; - const entry = '/path/to/function'; +```ts +const entry = '/path/to/function'; new python.PythonFunction(this, 'function', { entry, runtime: Runtime.PYTHON_3_8, bundling: { bundlingFileAccess: BundlingFileAccess.VOLUME_COPY, - }, + }, }); ``` diff --git a/packages/@aws-cdk/aws-lambda-python/rosetta/default.ts-fixture b/packages/@aws-cdk/aws-lambda-python/rosetta/default.ts-fixture index 45e67ca866edc..9d2d4ff5ee228 100644 --- a/packages/@aws-cdk/aws-lambda-python/rosetta/default.ts-fixture +++ b/packages/@aws-cdk/aws-lambda-python/rosetta/default.ts-fixture @@ -1,6 +1,6 @@ // Fixture with packages imported, but nothing else import { Construct } from 'constructs'; -import { DockerImage, Stack } from '@aws-cdk/core'; +import { DockerImage, Stack, BundlingFileAccess } from '@aws-cdk/core'; import { Runtime } from '@aws-cdk/aws-lambda'; import * as python from '@aws-cdk/aws-lambda-python'; diff --git a/packages/@aws-cdk/core/test/private/asset-staging.test.ts b/packages/@aws-cdk/core/test/private/asset-staging.test.ts index f45271297677e..31d316eed30af 100644 --- a/packages/@aws-cdk/core/test/private/asset-staging.test.ts +++ b/packages/@aws-cdk/core/test/private/asset-staging.test.ts @@ -6,7 +6,6 @@ import { AssetBundlingBindMount, AssetBundlingVolumeCopy } from '../../lib/priva describe('bundling', () => { afterEach(() => { sinon.restore(); - }); test('AssetBundlingVolumeCopy bundles with volume copy ', () => { @@ -109,7 +108,5 @@ describe('bundling', () => { '-v', 'alpine', ]), { stdio: ['ignore', process.stderr, 'inherit'] })).toEqual(true); - }); - }); diff --git a/packages/@aws-cdk/core/test/staging.test.ts b/packages/@aws-cdk/core/test/staging.test.ts index a13f23c495613..cbde1eaf0fc26 100644 --- a/packages/@aws-cdk/core/test/staging.test.ts +++ b/packages/@aws-cdk/core/test/staging.test.ts @@ -1,4 +1,3 @@ -// import * as child_process from 'child_process'; import * as os from 'os'; import * as path from 'path'; import { testDeprecated } from '@aws-cdk/cdk-build-tools';