Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(core): local bundling provider #9564

Merged
merged 17 commits into from
Aug 11, 2020
9 changes: 9 additions & 0 deletions packages/@aws-cdk/aws-lambda-nodejs/lib/bundling.ts
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,15 @@ export class Bundling {
? [{ containerPath: '/parcel-cache', hostPath: options.cacheDir }]
: [],
workingDirectory: path.dirname(containerEntryPath).replace(/\\/g, '/'), // Always use POSIX paths in the container
eladb marked this conversation as resolved.
Show resolved Hide resolved
runLocally: () => {
try {
require.resolve('parcel');
const { version } = require('parcel/package.json'); // eslint-disable-line import/no-extraneous-dependencies, @typescript-eslint/no-require-imports
return /^2/.test(version);
} catch {
return false;
}
},
},
});
}
Expand Down
1 change: 1 addition & 0 deletions packages/@aws-cdk/aws-lambda-nodejs/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@
"cdk-build-tools": "0.0.0",
"cdk-integ-tools": "0.0.0",
"delay": "4.3.0",
"parcel": "2.0.0-beta.1",
"pkglint": "0.0.0"
},
"dependencies": {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/* eslint-disable no-console */
import { S3 } from 'aws-sdk'; // eslint-disable-line import/no-extraneous-dependencies
import * as delay from 'delay';
import delay from 'delay';

const s3 = new S3();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
"Properties": {
"Code": {
"S3Bucket": {
"Ref": "AssetParametersf94126eb364c953df28028e19ee70b3d0c2ac3fa9b88d0f5475e95978cb2722eS3Bucket64177146"
"Ref": "AssetParametersc1bdb4a38711823a6e9b7a328551ed915a2464bcefa7bf9cd2417ad3276a72eeS3Bucket31F389EE"
},
"S3Key": {
"Fn::Join": [
Expand All @@ -49,7 +49,7 @@
"Fn::Split": [
"||",
{
"Ref": "AssetParametersf94126eb364c953df28028e19ee70b3d0c2ac3fa9b88d0f5475e95978cb2722eS3VersionKeyF8E0F956"
"Ref": "AssetParametersc1bdb4a38711823a6e9b7a328551ed915a2464bcefa7bf9cd2417ad3276a72eeS3VersionKey467CA932"
}
]
}
Expand All @@ -62,7 +62,7 @@
"Fn::Split": [
"||",
{
"Ref": "AssetParametersf94126eb364c953df28028e19ee70b3d0c2ac3fa9b88d0f5475e95978cb2722eS3VersionKeyF8E0F956"
"Ref": "AssetParametersc1bdb4a38711823a6e9b7a328551ed915a2464bcefa7bf9cd2417ad3276a72eeS3VersionKey467CA932"
}
]
}
Expand Down Expand Up @@ -92,17 +92,17 @@
}
},
"Parameters": {
"AssetParametersf94126eb364c953df28028e19ee70b3d0c2ac3fa9b88d0f5475e95978cb2722eS3Bucket64177146": {
"AssetParametersc1bdb4a38711823a6e9b7a328551ed915a2464bcefa7bf9cd2417ad3276a72eeS3Bucket31F389EE": {
"Type": "String",
"Description": "S3 bucket for asset \"f94126eb364c953df28028e19ee70b3d0c2ac3fa9b88d0f5475e95978cb2722e\""
"Description": "S3 bucket for asset \"c1bdb4a38711823a6e9b7a328551ed915a2464bcefa7bf9cd2417ad3276a72ee\""
},
"AssetParametersf94126eb364c953df28028e19ee70b3d0c2ac3fa9b88d0f5475e95978cb2722eS3VersionKeyF8E0F956": {
"AssetParametersc1bdb4a38711823a6e9b7a328551ed915a2464bcefa7bf9cd2417ad3276a72eeS3VersionKey467CA932": {
"Type": "String",
"Description": "S3 key for asset version \"f94126eb364c953df28028e19ee70b3d0c2ac3fa9b88d0f5475e95978cb2722e\""
"Description": "S3 key for asset version \"c1bdb4a38711823a6e9b7a328551ed915a2464bcefa7bf9cd2417ad3276a72ee\""
},
"AssetParametersf94126eb364c953df28028e19ee70b3d0c2ac3fa9b88d0f5475e95978cb2722eArtifactHash8BE4F210": {
"AssetParametersc1bdb4a38711823a6e9b7a328551ed915a2464bcefa7bf9cd2417ad3276a72eeArtifactHashB4A0FA40": {
"Type": "String",
"Description": "Artifact hash for asset \"f94126eb364c953df28028e19ee70b3d0c2ac3fa9b88d0f5475e95978cb2722e\""
"Description": "Artifact hash for asset \"c1bdb4a38711823a6e9b7a328551ed915a2464bcefa7bf9cd2417ad3276a72ee\""
}
}
}
7 changes: 4 additions & 3 deletions packages/@aws-cdk/core/lib/asset-staging.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@ import * as path from 'path';
import * as cxapi from '@aws-cdk/cx-api';
import * as fs from 'fs-extra';
import { AssetHashType, AssetOptions } from './assets';
import { BundlingOptions } from './bundling';
import { IBundlingOptions } from './bundling';
import { Construct } from './construct-compat';
import { FileSystem, FingerprintOptions } from './fs';
import { Stage } from './stage';
import { Construct } from './construct-compat';

const STAGING_TMP = '.cdk.staging';

Expand Down Expand Up @@ -145,7 +145,7 @@ export class AssetStaging extends Construct {
}
}

private bundle(options: BundlingOptions): string {
private bundle(options: IBundlingOptions): string {
// Temp staging directory in the working directory
const stagingTmp = path.join('.', STAGING_TMP);
fs.ensureDirSync(stagingTmp);
Expand Down Expand Up @@ -186,6 +186,7 @@ export class AssetStaging extends Construct {
volumes,
environment: options.environment,
workingDirectory: options.workingDirectory ?? AssetStaging.BUNDLING_INPUT_DIR,
runLocally: options.runLocally,
});
} catch (err) {
throw new Error(`Failed to run bundling Docker image for asset ${this.node.path}: ${err}`);
Expand Down
4 changes: 2 additions & 2 deletions packages/@aws-cdk/core/lib/assets.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { BundlingOptions } from './bundling';
import { IBundlingOptions } from './bundling';

/**
* Common interface for all assets.
Expand Down Expand Up @@ -55,7 +55,7 @@ export interface AssetOptions {
*
* @experimental
*/
readonly bundling?: BundlingOptions;
readonly bundling?: IBundlingOptions;
}

/**
Expand Down
74 changes: 51 additions & 23 deletions packages/@aws-cdk/core/lib/bundling.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { spawnSync, SpawnSyncOptions } from 'child_process';
*
* @experimental
*/
export interface BundlingOptions {
export interface IBundlingOptions {
/**
* The Docker image where the command will run.
*/
Expand Down Expand Up @@ -53,6 +53,12 @@ export interface BundlingOptions {
* @default - uid:gid of the current user or 1000:1000 on Windows
*/
readonly user?: string;

/**
* A user function to indicate if bundling should run locally instead
* of running in a Docker container.
*/
runLocally?(): boolean;
jogold marked this conversation as resolved.
Show resolved Hide resolved
}

/**
Expand Down Expand Up @@ -83,7 +89,7 @@ export class BundlingDockerImage {
path,
];

const docker = dockerExec(dockerArgs);
const docker = exec(process.env.CDK_DOCKER ?? 'docker', dockerArgs);

const match = docker.stdout.toString().match(/sha256:([a-z0-9]+)/);

Expand All @@ -102,32 +108,53 @@ export class BundlingDockerImage {
*
* @internal
*/
public _run(options: DockerRunOptions = {}) {
public _run(options: IDockerRunOptions = {}) {
const volumes = options.volumes || [];
const environment = options.environment || {};
const command = options.command || [];

const dockerArgs: string[] = [
'run', '--rm',
...options.user
? ['-u', options.user]
: [],
...flatten(volumes.map(v => ['-v', `${v.hostPath}:${v.containerPath}:${v.consistency ?? DockerVolumeConsistency.DELEGATED}`])),
...flatten(Object.entries(environment).map(([k, v]) => ['--env', `${k}=${v}`])),
...options.workingDirectory
? ['-w', options.workingDirectory]
: [],
this.image,
...command,
];

dockerExec(dockerArgs, {
stdio: [ // show Docker output
const spawnSyncOptions: SpawnSyncOptions = {
stdio: [ // show output
'ignore', // ignore stdio
process.stderr, // redirect stdout to stderr
'inherit', // inherit stderr
],
});
};

if (!options.runLocally || options.runLocally() === false) {
const dockerArgs: string[] = [
'run', '--rm',
...options.user
? ['-u', options.user]
: [],
...flatten(volumes.map(v => ['-v', `${v.hostPath}:${v.containerPath}:${v.consistency ?? DockerVolumeConsistency.DELEGATED}`])),
...flatten(Object.entries(environment).map(([k, v]) => ['--env', `${k}=${v}`])),
...options.workingDirectory
? ['-w', options.workingDirectory]
: [],
this.image,
...command,
];

exec(process.env.CDK_DOCKER ?? 'docker', dockerArgs, spawnSyncOptions);
}
else {
const [prog, ...args] = command.map(comm => {
let newCommand = comm;
for (const volume of volumes) {
newCommand = newCommand.replace(new RegExp(volume.containerPath, 'g'), volume.hostPath);
}
return newCommand;
});

exec(prog, args, {
env: {
...process.env,
...environment,
},
...spawnSyncOptions,
});
}
}
}

Expand Down Expand Up @@ -175,7 +202,7 @@ export enum DockerVolumeConsistency {
/**
* Docker run options
*/
interface DockerRunOptions {
interface IDockerRunOptions {
/**
* The command to run in the container.
*
Expand Down Expand Up @@ -210,6 +237,8 @@ interface DockerRunOptions {
* @default - root or image default
*/
readonly user?: string;

runLocally?(): boolean;
}

/**
Expand All @@ -228,8 +257,7 @@ function flatten(x: string[][]) {
return Array.prototype.concat([], ...x);
}

function dockerExec(args: string[], options?: SpawnSyncOptions) {
const prog = process.env.CDK_DOCKER ?? 'docker';
function exec(prog: string, args: string[], options?: SpawnSyncOptions) {
const proc = spawnSync(prog, args, options);

if (proc.error) {
Expand Down
Loading