diff --git a/lerna.json b/lerna.json index 8ea88bab3ad9c..82d242b045937 100644 --- a/lerna.json +++ b/lerna.json @@ -3,6 +3,7 @@ "packages": [ "packages/aws-cdk-lib", "packages/cdk-assets", + "packages/cdk-bogus", "packages/aws-cdk", "packages/cdk", "packages/@aws-cdk/*", diff --git a/packages/cdk-bogus/.eslintrc.js b/packages/cdk-bogus/.eslintrc.js new file mode 100644 index 0000000000000..2658ee8727166 --- /dev/null +++ b/packages/cdk-bogus/.eslintrc.js @@ -0,0 +1,3 @@ +const baseConfig = require('@aws-cdk/cdk-build-tools/config/eslintrc'); +baseConfig.parserOptions.project = __dirname + '/tsconfig.json'; +module.exports = baseConfig; diff --git a/packages/cdk-bogus/.gitignore b/packages/cdk-bogus/.gitignore new file mode 100644 index 0000000000000..d24092a6feda2 --- /dev/null +++ b/packages/cdk-bogus/.gitignore @@ -0,0 +1,28 @@ +*.js +*.js.map +*.d.ts +!lib/init-templates/**/javascript/**/* +node_modules +dist + +# Generated by generate.sh +build-info.json + +.LAST_BUILD +.nyc_output +coverage +nyc.config.js +.LAST_PACKAGE +*.snk + +!test/integ/run-wrappers/dist +!test/integ/cli/**/* +assets.json +npm-shrinkwrap.json +!.eslintrc.js +!jest.config.js + +junit.xml + +# Ignore this symlink, we recreate it at test time +test/test-archive-follow/data/linked diff --git a/packages/cdk-bogus/.npmignore b/packages/cdk-bogus/.npmignore new file mode 100644 index 0000000000000..45b8808bdd7ac --- /dev/null +++ b/packages/cdk-bogus/.npmignore @@ -0,0 +1,30 @@ +# Don't include original .ts files when doing `npm pack` +*.ts +!*.template.ts +!*.d.ts +coverage +.nyc_output +*.tgz + +dist +.LAST_PACKAGE +.LAST_BUILD +*.snk + +!lib/init-templates/*/*/tsconfig.json +!test/integ/cli/**/*.js +!test/integ/run-wrappers/dist + +*.tsbuildinfo + +tsconfig.json + +# init templates include default tsconfig.json files which we need +!lib/init-templates/**/tsconfig.json +.eslintrc.js +jest.config.js + +# exclude cdk artifacts +**/cdk.out +junit.xml +test/ \ No newline at end of file diff --git a/packages/cdk-bogus/LICENSE b/packages/cdk-bogus/LICENSE new file mode 100644 index 0000000000000..9b722c65c5481 --- /dev/null +++ b/packages/cdk-bogus/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2018-2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/packages/cdk-bogus/NOTICE b/packages/cdk-bogus/NOTICE new file mode 100644 index 0000000000000..a27b7dd317649 --- /dev/null +++ b/packages/cdk-bogus/NOTICE @@ -0,0 +1,2 @@ +AWS Cloud Development Kit (AWS CDK) +Copyright 2018-2023 Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/packages/cdk-bogus/README.md b/packages/cdk-bogus/README.md new file mode 100644 index 0000000000000..7c8bc78aca51b --- /dev/null +++ b/packages/cdk-bogus/README.md @@ -0,0 +1,190 @@ +# cdk-assets + + +--- + +![cdk-constructs: Stable](https://img.shields.io/badge/cdk--constructs-stable-success.svg?style=for-the-badge) + +--- + + + + +A tool for publishing CDK assets to AWS environments. + +## Overview + +`cdk-assets` requires an asset manifest file called `assets.json`, in a CDK +CloudAssembly (`cdk.out/assets.json`). It will take the assets listed in the +manifest, prepare them as required and upload them to the locations indicated in +the manifest. + +Currently the following asset types are supported: + +* Files and archives, uploaded to S3 +* Docker Images, uploaded to ECR +* Files, archives, and Docker images built by external utilities + +S3 buckets and ECR repositories to upload to are expected to exist already. + +We expect assets to be immutable, and we expect that immutability to be +reflected both in the asset ID and in the destination location. This reflects +itself in the following behaviors: + +* If the indicated asset already exists in the given destination location, it + will not be packaged and uploaded. +* If some locally cached artifact (depending on the asset type a file or an + image in the local Docker cache) already exists named after the asset's ID, it + will not be packaged, but will be uploaded directly to the destination + location. + +For assets build by external utilities, the contract is such that cdk-assets +expects the utility to manage dedupe detection as well as path/image tag generation. +This means that cdk-assets will call the external utility every time generation +is warranted, and it is up to the utility to a) determine whether to do a +full rebuild; and b) to return only one thing on stdout: the path to the file/archive +asset, or the name of the local Docker image. + +## Usage + +The `cdk-asset` tool can be used programmatically and via the CLI. Use +programmatic access if you need more control over authentication than the +default [`aws-sdk`](https://github.com/aws/aws-sdk-js) implementation allows. + +Command-line use looks like this: + +```console +$ cdk-assets /path/to/cdk.out [ASSET:DEST] [ASSET] [:DEST] [...] +``` + +Credentials will be taken from the `AWS_ACCESS_KEY...` environment variables +or the `default` profile (or another profile if `AWS_PROFILE` is set). + +A subset of the assets and destinations can be uploaded by specifying their +asset IDs or destination IDs. + +## Manifest Example + +An asset manifest looks like this: + +```json +{ + "version": "1.22.0", + "files": { + "7aac5b80b050e7e4e168f84feffa5893": { + "source": { + "path": "some_directory", + "packaging": "zip" + }, + "destinations": { + "us-east-1": { + "region": "us-east-1", + "assumeRoleArn": "arn:aws:iam::12345789012:role/my-account", + "bucketName": "MyBucket", + "objectKey": "7aac5b80b050e7e4e168f84feffa5893.zip" + } + } + }, + "3dfe2b80b050e7e4e168f84feff678d4": { + "source": { + "executable": ["myzip"] + }, + "destinations": { + "us-east-1": { + "region": "us-east-1", + "assumeRoleArn": "arn:aws:iam::12345789012:role/my-account", + "bucketName": "MySpecialBucket", + "objectKey": "3dfe2b80b050e7e4e168f84feff678d4.zip" + } + } + }, + }, + "dockerImages": { + "b48783c58a86f7b8c68a4591c4f9be31": { + "source": { + "directory": "dockerdir", + }, + "destinations": { + "us-east-1": { + "region": "us-east-1", + "assumeRoleArn": "arn:aws:iam::12345789012:role/my-account", + "repositoryName": "MyRepository", + "imageTag": "b48783c58a86f7b8c68a4591c4f9be31", + "imageUri": "123456789012.dkr.ecr.us-east-1.amazonaws.com/MyRepository:1234567891b48783c58a86f7b8c68a4591c4f9be31", + } + } + }, + "d92753c58a86f7b8c68a4591c4f9cf28": { + "source": { + "executable": ["mytool", "package", "dockerdir"], + }, + "destinations": { + "us-east-1": { + "region": "us-east-1", + "assumeRoleArn": "arn:aws:iam::12345789012:role/my-account", + "repositoryName": "MyRepository2", + "imageTag": "d92753c58a86f7b8c68a4591c4f9cf28", + "imageUri": "123456789987.dkr.ecr.us-east-1.amazonaws.com/MyRepository2:1234567891b48783c58a86f7b8c68a4591c4f9be31", + } + } + } + } +} +``` + +### Placeholders + +The `destination` block of an asset manifest may contain the following region +and account placeholders: + +* `${AWS::Region}` +* `${AWS::AccountId}` + +These will be substituted with the region and account IDs currently configured +on the AWS SDK (through environment variables or `~/.aws/...` config files). + +* The `${AWS::AccountId}` placeholder will *not* be re-evaluated after + performing the `AssumeRole` call. +* If `${AWS::Region}` is used, it will principally be replaced with the value + in the `region` key. If the default region is intended, leave the `region` + key out of the manifest at all. + +## Docker image credentials + +For Docker image asset publishing, `cdk-assets` will `docker login` with +credentials from ECR GetAuthorizationToken prior to building and publishing, so +that the Dockerfile can reference images in the account's ECR repo. + +`cdk-assets` can also be configured to read credentials from both ECR and +SecretsManager prior to build by creating a credential configuration at +'~/.cdk/cdk-docker-creds.json' (override this location by setting the +CDK_DOCKER_CREDS_FILE environment variable). The credentials file has the +following format: + +```json +{ + "version": "1.0", + "domainCredentials": { + "domain1.example.com": { + "secretsManagerSecretId": "mySecret", // Can be the secret ID or full ARN + "roleArn": "arn:aws:iam::0123456789012:role/my-role" // (Optional) role with permissions to the secret + }, + "domain2.example.com": { + "ecrRepository": true, + "roleArn": "arn:aws:iam::0123456789012:role/my-role" // (Optional) role with permissions to the repo + } + } +} +``` + +If the credentials file is present, `docker` will be configured to use the +`docker-credential-cdk-assets` credential helper for each of the domains listed +in the file. This helper will assume the role provided (if present), and then fetch +the login credentials from either SecretsManager or ECR. + +## Using Drop-in Docker Replacements + +By default, the AWS CDK will build and publish Docker image assets using the +`docker` command. However, by specifying the `CDK_DOCKER` environment variable, +you can override the command that will be used to build and publish your +assets. diff --git a/packages/cdk-bogus/bin/cdk-assets b/packages/cdk-bogus/bin/cdk-assets new file mode 100755 index 0000000000000..09c08dd446846 --- /dev/null +++ b/packages/cdk-bogus/bin/cdk-assets @@ -0,0 +1,2 @@ +#!/usr/bin/env node +require('./cdk-assets.js'); \ No newline at end of file diff --git a/packages/cdk-bogus/bin/cdk-assets.ts b/packages/cdk-bogus/bin/cdk-assets.ts new file mode 100644 index 0000000000000..4547051334449 --- /dev/null +++ b/packages/cdk-bogus/bin/cdk-assets.ts @@ -0,0 +1,67 @@ +import * as yargs from 'yargs'; +import { list } from './list'; +import { setLogThreshold, VERSION } from './logging'; +import { publish } from './publish'; +import { AssetManifest } from '../lib'; + +async function main() { + const argv = yargs + .usage('$0 [args]') + .option('verbose', { + alias: 'v', + type: 'boolean', + desc: 'Increase logging verbosity', + count: true, + default: 0, + }) + .option('path', { + alias: 'p', + type: 'string', + desc: 'The path (file or directory) to load the assets from. If a directory, ' + + `the file '${AssetManifest.DEFAULT_FILENAME}' will be loaded from it.`, + default: '.', + requiresArg: true, + }) + .command('ls', 'List assets from the given manifest', command => command + , wrapHandler(async args => { + await list(args); + })) + .command('publish [ASSET..]', 'Publish assets in the given manifest', command => command + .option('profile', { type: 'string', describe: 'Profile to use from AWS Credentials file' }) + .positional('ASSET', { type: 'string', array: true, describe: 'Assets to publish (format: "ASSET[:DEST]"), default all' }) + , wrapHandler(async args => { + await publish({ + path: args.path, + assets: args.ASSET, + profile: args.profile, + }); + })) + .demandCommand() + .help() + .strict() // Error on wrong command + .version(VERSION) + .showHelpOnFail(false) + .argv; + + // Evaluating .argv triggers the parsing but the command gets implicitly executed, + // so we don't need the output. + Array.isArray(argv); +} + +/** + * Wrap a command's handler with standard pre- and post-work + */ +function wrapHandler(handler: (x: A) => Promise) { + return async (argv: A) => { + if (argv.verbose) { + setLogThreshold('verbose'); + } + await handler(argv); + }; +} + +main().catch(e => { + // eslint-disable-next-line no-console + console.error(e.stack); + process.exitCode = 1; +}); diff --git a/packages/cdk-bogus/bin/docker-credential-cdk-assets b/packages/cdk-bogus/bin/docker-credential-cdk-assets new file mode 100755 index 0000000000000..3829057860102 --- /dev/null +++ b/packages/cdk-bogus/bin/docker-credential-cdk-assets @@ -0,0 +1,2 @@ +#!/usr/bin/env node +require('./docker-credential-cdk-assets.js'); diff --git a/packages/cdk-bogus/bin/docker-credential-cdk-assets.ts b/packages/cdk-bogus/bin/docker-credential-cdk-assets.ts new file mode 100644 index 0000000000000..6dccb5521cf55 --- /dev/null +++ b/packages/cdk-bogus/bin/docker-credential-cdk-assets.ts @@ -0,0 +1,42 @@ +/** + * Docker Credential Helper to retrieve credentials based on an external configuration file. + * Supports loading credentials from ECR repositories and from Secrets Manager, + * optionally via an assumed role. + * + * The only operation currently supported by this credential helper at this time is the `get` + * command, which receives a domain name as input on stdin and returns a Username/Secret in + * JSON format on stdout. + * + * IMPORTANT - The credential helper must not output anything else besides the final credentials + * in any success case; doing so breaks docker's parsing of the output and causes the login to fail. + */ + +import * as fs from 'fs'; +import { DefaultAwsClient } from '../lib'; + +import { cdkCredentialsConfig, cdkCredentialsConfigFile, fetchDockerLoginCredentials } from '../lib/private/docker-credentials'; + +async function main() { + // Expected invocation is [node, docker-credential-cdk-assets, get] with input fed via STDIN + // For other valid docker commands (store, list, erase), we no-op. + if (process.argv.length !== 3 || process.argv[2] !== 'get') { + process.exit(0); + } + + const config = cdkCredentialsConfig(); + if (!config) { + throw new Error(`unable to find CDK Docker credentials at: ${cdkCredentialsConfigFile()}`); + } + + // Read the domain to fetch from stdin + let endpoint = fs.readFileSync(0, { encoding: 'utf-8' }).trim(); + const credentials = await fetchDockerLoginCredentials(new DefaultAwsClient(), config, endpoint); + // Write the credentials back to stdout + fs.writeFileSync(1, JSON.stringify(credentials)); +} + +main().catch(e => { + // eslint-disable-next-line no-console + console.error(e.stack); + process.exitCode = 1; +}); diff --git a/packages/cdk-bogus/bin/list.ts b/packages/cdk-bogus/bin/list.ts new file mode 100644 index 0000000000000..e93358cd729fd --- /dev/null +++ b/packages/cdk-bogus/bin/list.ts @@ -0,0 +1,9 @@ +import { AssetManifest } from '../lib'; + +export async function list(args: { + path: string; +}) { + const manifest = AssetManifest.fromPath(args.path); + // eslint-disable-next-line no-console + console.log(manifest.list().join('\n')); +} \ No newline at end of file diff --git a/packages/cdk-bogus/bin/logging.ts b/packages/cdk-bogus/bin/logging.ts new file mode 100644 index 0000000000000..ead34deeaa70c --- /dev/null +++ b/packages/cdk-bogus/bin/logging.ts @@ -0,0 +1,24 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +export type LogLevel = 'verbose' | 'info' | 'error'; +let logThreshold: LogLevel = 'info'; + +export const VERSION = JSON.parse(fs.readFileSync(path.join(__dirname, '..', 'package.json'), { encoding: 'utf-8' })).version; + +const LOG_LEVELS: Record = { + verbose: 1, + info: 2, + error: 3, +}; + +export function setLogThreshold(threshold: LogLevel) { + logThreshold = threshold; +} + +export function log(level: LogLevel, message: string) { + if (LOG_LEVELS[level] >= LOG_LEVELS[logThreshold]) { + // eslint-disable-next-line no-console + console.error(`${level.padEnd(7, ' ')}: ${message}`); + } +} \ No newline at end of file diff --git a/packages/cdk-bogus/bin/publish.ts b/packages/cdk-bogus/bin/publish.ts new file mode 100644 index 0000000000000..87ead6eac14ae --- /dev/null +++ b/packages/cdk-bogus/bin/publish.ts @@ -0,0 +1,56 @@ +import { log, LogLevel } from './logging'; +import { + AssetManifest, AssetPublishing, DefaultAwsClient, DestinationPattern, EventType, + IPublishProgress, IPublishProgressListener, +} from '../lib'; + +export async function publish(args: { + path: string; + assets?: string[]; + profile?: string; +}) { + + let manifest = AssetManifest.fromPath(args.path); + log('verbose', `Loaded manifest from ${args.path}: ${manifest.entries.length} assets found`); + + if (args.assets && args.assets.length > 0) { + const selection = args.assets.map(a => DestinationPattern.parse(a)); + manifest = manifest.select(selection); + log('verbose', `Applied selection: ${manifest.entries.length} assets selected.`); + } + + const pub = new AssetPublishing(manifest, { + aws: new DefaultAwsClient(args.profile), + progressListener: new ConsoleProgress(), + throwOnError: false, + }); + + await pub.publish(); + + if (pub.hasFailures) { + for (const failure of pub.failures) { + // eslint-disable-next-line no-console + console.error('Failure:', failure.error.stack); + } + + process.exitCode = 1; + } +} + +const EVENT_TO_LEVEL: Record = { + build: 'verbose', + cached: 'verbose', + check: 'verbose', + debug: 'verbose', + fail: 'error', + found: 'verbose', + start: 'info', + success: 'info', + upload: 'verbose', +}; + +class ConsoleProgress implements IPublishProgressListener { + public onPublishEvent(type: EventType, event: IPublishProgress): void { + log(EVENT_TO_LEVEL[type], `[${event.percentComplete}%] ${type}: ${event.message}`); + } +} diff --git a/packages/cdk-bogus/jest.config.js b/packages/cdk-bogus/jest.config.js new file mode 100644 index 0000000000000..4147a830a714b --- /dev/null +++ b/packages/cdk-bogus/jest.config.js @@ -0,0 +1,11 @@ +const baseConfig = require('@aws-cdk/cdk-build-tools/config/jest.config'); +module.exports = { + ...baseConfig, + coverageThreshold: { + global: { + ...baseConfig.coverageThreshold.global, + statements: 75, + branches: 60, + }, + }, +}; diff --git a/packages/cdk-bogus/lib/asset-manifest.ts b/packages/cdk-bogus/lib/asset-manifest.ts new file mode 100644 index 0000000000000..857bbbc8b0144 --- /dev/null +++ b/packages/cdk-bogus/lib/asset-manifest.ts @@ -0,0 +1,313 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import { + AssetManifest as AssetManifestSchema, DockerImageDestination, DockerImageSource, + FileDestination, FileSource, Manifest, +} from '@aws-cdk/cloud-assembly-schema'; + +/** + * A manifest of assets + */ +export class AssetManifest { + /** + * The default name of the asset manifest in a cdk.out directory + */ + public static readonly DEFAULT_FILENAME = 'assets.json'; + + /** + * Load an asset manifest from the given file + */ + public static fromFile(fileName: string) { + try { + const obj = Manifest.loadAssetManifest(fileName); + return new AssetManifest(path.dirname(fileName), obj); + } catch (e: any) { + throw new Error(`Canot read asset manifest '${fileName}': ${e.message}`); + } + } + + /** + * Load an asset manifest from the given file or directory + * + * If the argument given is a directoy, the default asset file name will be used. + */ + public static fromPath(filePath: string) { + let st; + try { + st = fs.statSync(filePath); + } catch (e: any) { + throw new Error(`Cannot read asset manifest at '${filePath}': ${e.message}`); + } + if (st.isDirectory()) { + return AssetManifest.fromFile(path.join(filePath, AssetManifest.DEFAULT_FILENAME)); + } + return AssetManifest.fromFile(filePath); + } + + /** + * The directory where the manifest was found + */ + public readonly directory: string; + + constructor(directory: string, private readonly manifest: AssetManifestSchema) { + this.directory = directory; + } + + /** + * Select a subset of assets and destinations from this manifest. + * + * Only assets with at least 1 selected destination are retained. + * + * If selection is not given, everything is returned. + */ + public select(selection?: DestinationPattern[]): AssetManifest { + if (selection === undefined) { return this; } + + const ret: AssetManifestSchema & Required> + = { version: this.manifest.version, dockerImages: {}, files: {} }; + + for (const assetType of ASSET_TYPES) { + for (const [assetId, asset] of Object.entries(this.manifest[assetType] || {})) { + const filteredDestinations = filterDict( + asset.destinations, + (_, destId) => selection.some(sel => sel.matches(new DestinationIdentifier(assetId, destId)))); + + if (Object.keys(filteredDestinations).length > 0) { + ret[assetType][assetId] = { + ...asset, + destinations: filteredDestinations, + }; + } + } + } + + return new AssetManifest(this.directory, ret); + } + + /** + * Describe the asset manifest as a list of strings + */ + public list() { + return [ + ...describeAssets('file', this.manifest.files || {}), + ...describeAssets('docker-image', this.manifest.dockerImages || {}), + ]; + + function describeAssets(type: string, assets: Record }>) { + const ret = new Array(); + for (const [assetId, asset] of Object.entries(assets || {})) { + ret.push(`${assetId} ${type} ${JSON.stringify(asset.source)}`); + + const destStrings = Object.entries(asset.destinations).map(([destId, dest]) => ` ${assetId}:${destId} ${JSON.stringify(dest)}`); + ret.push(...prefixTreeChars(destStrings, ' ')); + } + return ret; + } + } + + /** + * List of assets per destination + * + * Returns one asset for every publishable destination. Multiple asset + * destinations may share the same asset source. + */ + public get entries(): IManifestEntry[] { + return [ + ...makeEntries(this.manifest.files || {}, FileManifestEntry), + ...makeEntries(this.manifest.dockerImages || {}, DockerImageManifestEntry), + ]; + } + + /** + * List of file assets, splat out to destinations + */ + public get files(): FileManifestEntry[] { + return makeEntries(this.manifest.files || {}, FileManifestEntry); + } +} + +function makeEntries( + assets: Record }>, + ctor: new (id: DestinationIdentifier, source: A, destination: B) => C): C[] { + + const ret = new Array(); + for (const [assetId, asset] of Object.entries(assets)) { + for (const [destId, destination] of Object.entries(asset.destinations)) { + ret.push(new ctor(new DestinationIdentifier(assetId, destId), asset.source, destination)); + } + } + return ret; +} + +type AssetType = 'files' | 'dockerImages'; + +const ASSET_TYPES: AssetType[] = ['files', 'dockerImages']; + +/** + * A single asset from an asset manifest' + */ +export interface IManifestEntry { + /** + * The identifier of the asset and its destination + */ + readonly id: DestinationIdentifier; + + /** + * The type of asset + */ + readonly type: string; + + /** + * Type-dependent source data + */ + readonly genericSource: unknown; + + /** + * Type-dependent destination data + */ + readonly genericDestination: unknown; +} + +/** + * A manifest entry for a file asset + */ +export class FileManifestEntry implements IManifestEntry { + public readonly genericSource: unknown; + public readonly genericDestination: unknown; + public readonly type = 'file'; + + constructor( + /** Identifier for this asset */ + public readonly id: DestinationIdentifier, + /** Source of the file asset */ + public readonly source: FileSource, + /** Destination for the file asset */ + public readonly destination: FileDestination, + ) { + this.genericSource = source; + this.genericDestination = destination; + } +} + +/** + * A manifest entry for a docker image asset + */ +export class DockerImageManifestEntry implements IManifestEntry { + public readonly genericSource: unknown; + public readonly genericDestination: unknown; + public readonly type = 'docker-image'; + + constructor( + /** Identifier for this asset */ + public readonly id: DestinationIdentifier, + /** Source of the file asset */ + public readonly source: DockerImageSource, + /** Destination for the file asset */ + public readonly destination: DockerImageDestination, + ) { + this.genericSource = source; + this.genericDestination = destination; + } +} + +/** + * Identify an asset destination in an asset manifest + * + * When stringified, this will be a combination of the source + * and destination IDs. + */ +export class DestinationIdentifier { + /** + * Identifies the asset, by source. + * + * The assetId will be the same between assets that represent + * the same physical file or image. + */ + public readonly assetId: string; + + /** + * Identifies the destination where this asset will be published + */ + public readonly destinationId: string; + + constructor(assetId: string, destinationId: string) { + this.assetId = assetId; + this.destinationId = destinationId; + } + + /** + * Return a string representation for this asset identifier + */ + public toString() { + return this.destinationId ? `${this.assetId}:${this.destinationId}` : this.assetId; + } +} + +function filterDict(xs: Record, pred: (x: A, key: string) => boolean): Record { + const ret: Record = {}; + for (const [key, value] of Object.entries(xs)) { + if (pred(value, key)) { + ret[key] = value; + } + } + return ret; +} + +/** + * A filter pattern for an destination identifier + */ +export class DestinationPattern { + /** + * Parse a ':'-separated string into an asset/destination identifier + */ + public static parse(s: string) { + if (!s) { throw new Error('Empty string is not a valid destination identifier'); } + const parts = s.split(':').map(x => x !== '*' ? x : undefined); + if (parts.length === 1) { return new DestinationPattern(parts[0]); } + if (parts.length === 2) { return new DestinationPattern(parts[0] || undefined, parts[1] || undefined); } + throw new Error(`Asset identifier must contain at most 2 ':'-separated parts, got '${s}'`); + } + + /** + * Identifies the asset, by source. + */ + public readonly assetId?: string; + + /** + * Identifies the destination where this asset will be published + */ + public readonly destinationId?: string; + + constructor(assetId?: string, destinationId?: string) { + this.assetId = assetId; + this.destinationId = destinationId; + } + + /** + * Whether or not this pattern matches the given identifier + */ + public matches(id: DestinationIdentifier) { + return (this.assetId === undefined || this.assetId === id.assetId) + && (this.destinationId === undefined || this.destinationId === id.destinationId); + } + + /** + * Return a string representation for this asset identifier + */ + public toString() { + return `${this.assetId ?? '*'}:${this.destinationId ?? '*'}`; + } +} + +/** + * Prefix box-drawing characters to make lines look like a hanging tree + */ +function prefixTreeChars(xs: string[], prefix = '') { + const ret = new Array(); + for (let i = 0; i < xs.length; i++) { + const isLast = i === xs.length - 1; + const boxChar = isLast ? '└' : '├'; + ret.push(`${prefix}${boxChar}${xs[i]}`); + } + return ret; +} diff --git a/packages/cdk-bogus/lib/aws.ts b/packages/cdk-bogus/lib/aws.ts new file mode 100644 index 0000000000000..d78e29f24cc3e --- /dev/null +++ b/packages/cdk-bogus/lib/aws.ts @@ -0,0 +1,163 @@ +import * as os from 'os'; + +/** + * AWS SDK operations required by Asset Publishing + */ +export interface IAws { + discoverPartition(): Promise; + discoverDefaultRegion(): Promise; + discoverCurrentAccount(): Promise; + + discoverTargetAccount(options: ClientOptions): Promise; + s3Client(options: ClientOptions): Promise; + ecrClient(options: ClientOptions): Promise; + secretsManagerClient(options: ClientOptions): Promise; +} + +export interface ClientOptions { + region?: string; + assumeRoleArn?: string; + assumeRoleExternalId?: string; + quiet?: boolean; +} + +/** + * An AWS account + * + * An AWS account always exists in only one partition. Usually we don't care about + * the partition, but when we need to form ARNs we do. + */ +export interface Account { + /** + * The account number + */ + readonly accountId: string; + + /** + * The partition ('aws' or 'aws-cn' or otherwise) + */ + readonly partition: string; +} + +/** + * AWS client using the AWS SDK for JS with no special configuration + */ +export class DefaultAwsClient implements IAws { + private readonly AWS: typeof import('aws-sdk'); + private account?: Account; + + constructor(profile?: string) { + // Force AWS SDK to look in ~/.aws/credentials and potentially use the configured profile. + process.env.AWS_SDK_LOAD_CONFIG = '1'; + process.env.AWS_STS_REGIONAL_ENDPOINTS = 'regional'; + process.env.AWS_NODEJS_CONNECTION_REUSE_ENABLED = '1'; + if (profile) { + process.env.AWS_PROFILE = profile; + } + // Stop SDKv2 from displaying a warning for now. We are aware and will migrate at some point, + // our customer don't need to be bothered with this. + process.env.AWS_SDK_JS_SUPPRESS_MAINTENANCE_MODE_MESSAGE = '1'; + + // We need to set the environment before we load this library for the first time. + // eslint-disable-next-line @typescript-eslint/no-require-imports + this.AWS = require('aws-sdk'); + } + + public async s3Client(options: ClientOptions) { + return new this.AWS.S3(await this.awsOptions(options)); + } + + public async ecrClient(options: ClientOptions) { + return new this.AWS.ECR(await this.awsOptions(options)); + } + + public async secretsManagerClient(options: ClientOptions) { + return new this.AWS.SecretsManager(await this.awsOptions(options)); + } + + public async discoverPartition(): Promise { + return (await this.discoverCurrentAccount()).partition; + } + + public async discoverDefaultRegion(): Promise { + return this.AWS.config.region || 'us-east-1'; + } + + public async discoverCurrentAccount(): Promise { + if (this.account === undefined) { + const sts = new this.AWS.STS(); + const response = await sts.getCallerIdentity().promise(); + if (!response.Account || !response.Arn) { + throw new Error(`Unrecognized response from STS: '${JSON.stringify(response)}'`); + } + this.account = { + accountId: response.Account!, + partition: response.Arn!.split(':')[1], + }; + } + + return this.account; + } + + public async discoverTargetAccount(options: ClientOptions): Promise { + const sts = new this.AWS.STS(await this.awsOptions(options)); + const response = await sts.getCallerIdentity().promise(); + if (!response.Account || !response.Arn) { + throw new Error(`Unrecognized response from STS: '${JSON.stringify(response)}'`); + } + return { + accountId: response.Account!, + partition: response.Arn!.split(':')[1], + }; + } + + private async awsOptions(options: ClientOptions) { + let credentials; + + if (options.assumeRoleArn) { + credentials = await this.assumeRole(options.region, options.assumeRoleArn, options.assumeRoleExternalId); + } + + return { + region: options.region, + customUserAgent: 'cdk-assets', + credentials, + }; + } + + /** + * Explicit manual AssumeRole call + * + * Necessary since I can't seem to get the built-in support for ChainableTemporaryCredentials to work. + * + * It needs an explicit configuration of `masterCredentials`, we need to put + * a `DefaultCredentialProverChain()` in there but that is not possible. + */ + private async assumeRole(region: string | undefined, roleArn: string, externalId?: string): Promise { + return new this.AWS.ChainableTemporaryCredentials({ + params: { + RoleArn: roleArn, + ExternalId: externalId, + RoleSessionName: `cdk-assets-${safeUsername()}`, + }, + stsConfig: { + region, + customUserAgent: 'cdk-assets', + }, + }); + } +} + +/** + * Return the username with characters invalid for a RoleSessionName removed + * + * @see https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html#API_AssumeRole_RequestParameters + */ +function safeUsername() { + try { + return os.userInfo().username.replace(/[^\w+=,.@-]/g, '@'); + } catch { + return 'noname'; + } +} + diff --git a/packages/cdk-bogus/lib/index.ts b/packages/cdk-bogus/lib/index.ts new file mode 100644 index 0000000000000..26f81852f3601 --- /dev/null +++ b/packages/cdk-bogus/lib/index.ts @@ -0,0 +1,4 @@ +export * from './publishing'; +export * from './asset-manifest'; +export * from './aws'; +export * from './progress'; diff --git a/packages/cdk-bogus/lib/private/archive.ts b/packages/cdk-bogus/lib/private/archive.ts new file mode 100644 index 0000000000000..8e0d9a900b46e --- /dev/null +++ b/packages/cdk-bogus/lib/private/archive.ts @@ -0,0 +1,92 @@ +import { createWriteStream, promises as fs } from 'fs'; +import * as path from 'path'; +import * as glob from 'glob'; + +// namespace object imports won't work in the bundle for function exports +// eslint-disable-next-line @typescript-eslint/no-require-imports +const archiver = require('archiver'); + +type Logger = (x: string) => void; + +export async function zipDirectory(directory: string, outputFile: string, logger: Logger): Promise { + // We write to a temporary file and rename at the last moment. This is so that if we are + // interrupted during this process, we don't leave a half-finished file in the target location. + const temporaryOutputFile = `${outputFile}.${randomString()}._tmp`; + await writeZipFile(directory, temporaryOutputFile); + await moveIntoPlace(temporaryOutputFile, outputFile, logger); +} + +function writeZipFile(directory: string, outputFile: string): Promise { + return new Promise(async (ok, fail) => { + // The below options are needed to support following symlinks when building zip files: + // - nodir: This will prevent symlinks themselves from being copied into the zip. + // - follow: This will follow symlinks and copy the files within. + const globOptions = { + dot: true, + nodir: true, + follow: true, + cwd: directory, + }; + const files = glob.sync('**', globOptions); // The output here is already sorted + + const output = createWriteStream(outputFile); + + const archive = archiver('zip'); + archive.on('warning', fail); + archive.on('error', fail); + + // archive has been finalized and the output file descriptor has closed, resolve promise + // this has to be done before calling `finalize` since the events may fire immediately after. + // see https://www.npmjs.com/package/archiver + output.once('close', ok); + + archive.pipe(output); + + // Append files serially to ensure file order + for (const file of files) { + const fullPath = path.resolve(directory, file); + const [data, stat] = await Promise.all([fs.readFile(fullPath), fs.stat(fullPath)]); + archive.append(data, { + name: file, + date: new Date('1980-01-01T00:00:00.000Z'), // reset dates to get the same hash for the same content + mode: stat.mode, + }); + } + + await archive.finalize(); + }); +} + +/** + * Rename the file to the target location, taking into account: + * + * - That we may see EPERM on Windows while an Antivirus scanner still has the + * file open, so retry a couple of times. + * - This same function may be called in parallel and be interrupted at any point. + */ +async function moveIntoPlace(source: string, target: string, logger: Logger) { + let delay = 100; + let attempts = 5; + while (true) { + try { + // 'rename' is guaranteed to overwrite an existing target, as long as it is a file (not a directory) + await fs.rename(source, target); + return; + } catch (e: any) { + if (e.code !== 'EPERM' || attempts-- <= 0) { + throw e; + } + logger(e.message); + await sleep(Math.floor(Math.random() * delay)); + delay *= 2; + } + } +} + +function sleep(ms: number) { + return new Promise(ok => setTimeout(ok, ms)); +} + +function randomString() { + return Math.random().toString(36).replace(/[^a-z0-9]+/g, ''); +} diff --git a/packages/cdk-bogus/lib/private/asset-handler.ts b/packages/cdk-bogus/lib/private/asset-handler.ts new file mode 100644 index 0000000000000..5baa32d9d52f4 --- /dev/null +++ b/packages/cdk-bogus/lib/private/asset-handler.ts @@ -0,0 +1,31 @@ +import { DockerFactory } from './docker'; +import { IAws } from '../aws'; +import { EventType } from '../progress'; + +/** + * Handler for asset building and publishing. + */ +export interface IAssetHandler { + /** + * Build the asset. + */ + build(): Promise; + + /** + * Publish the asset. + */ + publish(): Promise; + + /** + * Return whether the asset already exists + */ + isPublished(): Promise; +} + +export interface IHandlerHost { + readonly aws: IAws; + readonly aborted: boolean; + readonly dockerFactory: DockerFactory; + + emitMessage(type: EventType, m: string): void; +} \ No newline at end of file diff --git a/packages/cdk-bogus/lib/private/docker-credentials.ts b/packages/cdk-bogus/lib/private/docker-credentials.ts new file mode 100644 index 0000000000000..c46add8caeefb --- /dev/null +++ b/packages/cdk-bogus/lib/private/docker-credentials.ts @@ -0,0 +1,93 @@ +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { Logger } from './shell'; +import { IAws } from '../aws'; + +export interface DockerCredentials { + readonly Username: string; + readonly Secret: string; +} + +export interface DockerCredentialsConfig { + readonly version: string; + readonly domainCredentials: Record; +} + +export interface DockerDomainCredentialSource { + readonly secretsManagerSecretId?: string; + readonly secretsUsernameField?: string; + readonly secretsPasswordField?: string; + readonly ecrRepository?: boolean; + readonly assumeRoleArn?: string; +} + +/** Returns the presumed location of the CDK Docker credentials config file */ +export function cdkCredentialsConfigFile(): string { + return process.env.CDK_DOCKER_CREDS_FILE ?? path.join((os.userInfo().homedir ?? os.homedir()).trim() || '/', '.cdk', 'cdk-docker-creds.json'); +} + +let _cdkCredentials: DockerCredentialsConfig | undefined; +/** Loads and parses the CDK Docker credentials configuration, if it exists. */ +export function cdkCredentialsConfig(): DockerCredentialsConfig | undefined { + if (!_cdkCredentials) { + try { + _cdkCredentials = JSON.parse(fs.readFileSync(cdkCredentialsConfigFile(), { encoding: 'utf-8' })) as DockerCredentialsConfig; + } catch { } + } + return _cdkCredentials; +} + +/** Fetches login credentials from the configured source (e.g., SecretsManager, ECR) */ +export async function fetchDockerLoginCredentials(aws: IAws, config: DockerCredentialsConfig, endpoint: string) { + // Paranoid handling to ensure new URL() doesn't throw if the schema is missing + // For official docker registry, docker will pass https://index.docker.io/v1/ + endpoint = endpoint.includes('://') ? endpoint : `https://${endpoint}`; + const domain = new URL(endpoint).hostname; + + if (!Object.keys(config.domainCredentials).includes(domain) && !Object.keys(config.domainCredentials).includes(endpoint)) { + throw new Error(`unknown domain ${domain}`); + } + + let domainConfig = config.domainCredentials[domain] ?? config.domainCredentials[endpoint]; + + if (domainConfig.secretsManagerSecretId) { + const sm = await aws.secretsManagerClient({ assumeRoleArn: domainConfig.assumeRoleArn }); + const secretValue = await sm.getSecretValue({ SecretId: domainConfig.secretsManagerSecretId }).promise(); + if (!secretValue.SecretString) { throw new Error(`unable to fetch SecretString from secret: ${domainConfig.secretsManagerSecretId}`); }; + + const secret = JSON.parse(secretValue.SecretString); + + const usernameField = domainConfig.secretsUsernameField ?? 'username'; + const secretField = domainConfig.secretsPasswordField ?? 'secret'; + if (!secret[usernameField] || !secret[secretField]) { + throw new Error(`malformed secret string ("${usernameField}" or "${secretField}" field missing)`); + } + + return { Username: secret[usernameField], Secret: secret[secretField] }; + } else if (domainConfig.ecrRepository) { + const ecr = await aws.ecrClient({ assumeRoleArn: domainConfig.assumeRoleArn }); + const ecrAuthData = await obtainEcrCredentials(ecr); + + return { Username: ecrAuthData.username, Secret: ecrAuthData.password }; + } else { + throw new Error('unknown credential type: no secret ID or ECR repo'); + } +} + +export async function obtainEcrCredentials(ecr: AWS.ECR, logger?: Logger) { + if (logger) { logger('Fetching ECR authorization token'); } + const authData = (await ecr.getAuthorizationToken({ }).promise()).authorizationData || []; + if (authData.length === 0) { + throw new Error('No authorization data received from ECR'); + } + const token = Buffer.from(authData[0].authorizationToken!, 'base64').toString('ascii'); + const [username, password] = token.split(':'); + if (!username || !password) { throw new Error('unexpected ECR authData format'); } + + return { + username, + password, + endpoint: authData[0].proxyEndpoint!, + }; +} diff --git a/packages/cdk-bogus/lib/private/docker.ts b/packages/cdk-bogus/lib/private/docker.ts new file mode 100644 index 0000000000000..4dae410047497 --- /dev/null +++ b/packages/cdk-bogus/lib/private/docker.ts @@ -0,0 +1,266 @@ +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { cdkCredentialsConfig, obtainEcrCredentials } from './docker-credentials'; +import { Logger, shell, ShellOptions, ProcessFailedError } from './shell'; +import { createCriticalSection } from './util'; + +interface BuildOptions { + readonly directory: string; + + /** + * Tag the image with a given repoName:tag combination + */ + readonly tag: string; + readonly target?: string; + readonly file?: string; + readonly buildArgs?: Record; + readonly buildSecrets?: Record; + readonly networkMode?: string; + readonly platform?: string; + readonly outputs?: string[]; + readonly cacheFrom?: DockerCacheOption[]; + readonly cacheTo?: DockerCacheOption; +} + +export interface DockerCredentialsConfig { + readonly version: string; + readonly domainCredentials: Record; +} + +export interface DockerDomainCredentials { + readonly secretsManagerSecretId?: string; + readonly ecrRepository?: string; +} + +enum InspectImageErrorCode { + Docker = 1, + Podman = 125 +} + +export interface DockerCacheOption { + readonly type: string; + readonly params?: { [key: string]: string }; +} + +export class Docker { + + private configDir: string | undefined = undefined; + + constructor(private readonly logger?: Logger) { + } + + /** + * Whether an image with the given tag exists + */ + public async exists(tag: string) { + try { + await this.execute(['inspect', tag], { quiet: true }); + return true; + } catch (e: any) { + const error: ProcessFailedError = e; + + /** + * The only error we expect to be thrown will have this property and value. + * If it doesn't, it's unrecognized so re-throw it. + */ + if (error.code !== 'PROCESS_FAILED') { + throw error; + } + + /** + * If we know the shell command above returned an error, check to see + * if the exit code is one we know to actually mean that the image doesn't + * exist. + */ + switch (error.exitCode) { + case InspectImageErrorCode.Docker: + case InspectImageErrorCode.Podman: + // Docker and Podman will return this exit code when an image doesn't exist, return false + // context: https://github.com/aws/aws-cdk/issues/16209 + return false; + default: + // This is an error but it's not an exit code we recognize, throw. + throw error; + } + } + } + + public async build(options: BuildOptions) { + const buildCommand = [ + 'build', + ...flatten(Object.entries(options.buildArgs || {}).map(([k, v]) => ['--build-arg', `${k}=${v}`])), + ...flatten(Object.entries(options.buildSecrets || {}).map(([k, v]) => ['--secret', `id=${k},${v}`])), + '--tag', options.tag, + ...options.target ? ['--target', options.target] : [], + ...options.file ? ['--file', options.file] : [], + ...options.networkMode ? ['--network', options.networkMode] : [], + ...options.platform ? ['--platform', options.platform] : [], + ...options.outputs ? options.outputs.map(output => [`--output=${output}`]) : [], + ...options.cacheFrom ? [...options.cacheFrom.map(cacheFrom => ['--cache-from', this.cacheOptionToFlag(cacheFrom)]).flat()] : [], + ...options.cacheTo ? ['--cache-to', this.cacheOptionToFlag(options.cacheTo)] : [], + '.', + ]; + await this.execute(buildCommand, { cwd: options.directory }); + } + + /** + * Get credentials from ECR and run docker login + */ + public async login(ecr: AWS.ECR) { + const credentials = await obtainEcrCredentials(ecr); + + // Use --password-stdin otherwise docker will complain. Loudly. + await this.execute(['login', + '--username', credentials.username, + '--password-stdin', + credentials.endpoint], { + input: credentials.password, + + // Need to quiet otherwise Docker will complain + // 'WARNING! Your password will be stored unencrypted' + // doesn't really matter since it's a token. + quiet: true, + }); + } + + public async tag(sourceTag: string, targetTag: string) { + await this.execute(['tag', sourceTag, targetTag]); + } + + public async push(tag: string) { + await this.execute(['push', tag]); + } + + /** + * If a CDK Docker Credentials file exists, creates a new Docker config directory. + * Sets up `docker-credential-cdk-assets` to be the credential helper for each domain in the CDK config. + * All future commands (e.g., `build`, `push`) will use this config. + * + * See https://docs.docker.com/engine/reference/commandline/login/#credential-helpers for more details on cred helpers. + * + * @returns true if CDK config was found and configured, false otherwise + */ + public configureCdkCredentials(): boolean { + const config = cdkCredentialsConfig(); + if (!config) { return false; } + + this.configDir = fs.mkdtempSync(path.join(os.tmpdir(), 'cdkDockerConfig')); + + const domains = Object.keys(config.domainCredentials); + const credHelpers = domains.reduce((map: Record, domain) => { + map[domain] = 'cdk-assets'; // Use docker-credential-cdk-assets for this domain + return map; + }, {}); + fs.writeFileSync(path.join(this.configDir, 'config.json'), JSON.stringify({ credHelpers }), { encoding: 'utf-8' }); + + return true; + } + + /** + * Removes any configured Docker config directory. + * All future commands (e.g., `build`, `push`) will use the default config. + * + * This is useful after calling `configureCdkCredentials` to reset to default credentials. + */ + public resetAuthPlugins() { + this.configDir = undefined; + } + + private async execute(args: string[], options: ShellOptions = {}) { + const configArgs = this.configDir ? ['--config', this.configDir] : []; + + const pathToCdkAssets = path.resolve(__dirname, '..', '..', 'bin'); + try { + await shell([getDockerCmd(), ...configArgs, ...args], { + logger: this.logger, + ...options, + env: { + ...process.env, + ...options.env, + PATH: `${pathToCdkAssets}${path.delimiter}${options.env?.PATH ?? process.env.PATH}`, + }, + }); + } catch (e: any) { + if (e.code === 'ENOENT') { + throw new Error('Unable to execute \'docker\' in order to build a container asset. Please install \'docker\' and try again.'); + } + throw e; + } + } + + private cacheOptionToFlag(option: DockerCacheOption): string { + let flag = `type=${option.type}`; + if (option.params) { + flag += ',' + Object.entries(option.params).map(([k, v]) => `${k}=${v}`).join(','); + } + return flag; + } +} + +export interface DockerFactoryOptions { + readonly repoUri: string; + readonly ecr: AWS.ECR; + readonly logger: (m: string) => void; +} + +/** + * Helps get appropriately configured Docker instances during the container + * image publishing process. + */ +export class DockerFactory { + private enterLoggedInDestinationsCriticalSection = createCriticalSection(); + private loggedInDestinations = new Set(); + + /** + * Gets a Docker instance for building images. + */ + public async forBuild(options: DockerFactoryOptions): Promise { + const docker = new Docker(options.logger); + + // Default behavior is to login before build so that the Dockerfile can reference images in the ECR repo + // However, if we're in a pipelines environment (for example), + // we may have alternative credentials to the default ones to use for the build itself. + // If the special config file is present, delay the login to the default credentials until the push. + // If the config file is present, we will configure and use those credentials for the build. + let cdkDockerCredentialsConfigured = docker.configureCdkCredentials(); + if (!cdkDockerCredentialsConfigured) { + await this.loginOncePerDestination(docker, options); + } + + return docker; + } + + /** + * Gets a Docker instance for pushing images to ECR. + */ + public async forEcrPush(options: DockerFactoryOptions) { + const docker = new Docker(options.logger); + await this.loginOncePerDestination(docker, options); + return docker; + } + + private async loginOncePerDestination(docker: Docker, options: DockerFactoryOptions) { + // Changes: 012345678910.dkr.ecr.us-west-2.amazonaws.com/tagging-test + // To this: 012345678910.dkr.ecr.us-west-2.amazonaws.com + const repositoryDomain = options.repoUri.split('/')[0]; + + // Ensure one-at-a-time access to loggedInDestinations. + await this.enterLoggedInDestinationsCriticalSection(async () => { + if (this.loggedInDestinations.has(repositoryDomain)) { + return; + } + + await docker.login(options.ecr); + this.loggedInDestinations.add(repositoryDomain); + }); + } +} + +function getDockerCmd(): string { + return process.env.CDK_DOCKER ?? 'docker'; +} + +function flatten(x: string[][]) { + return Array.prototype.concat([], ...x); +} diff --git a/packages/cdk-bogus/lib/private/fs-extra.ts b/packages/cdk-bogus/lib/private/fs-extra.ts new file mode 100644 index 0000000000000..ac865789eb269 --- /dev/null +++ b/packages/cdk-bogus/lib/private/fs-extra.ts @@ -0,0 +1,31 @@ +import * as fs from 'fs'; +import * as path from 'path'; + +const pfs = fs.promises; + +export async function pathExists(pathName: string) { + try { + await pfs.stat(pathName); + return true; + } catch (e: any) { + if (e.code !== 'ENOENT') { throw e; } + return false; + } +} + +export function emptyDirSync(dir: string) { + fs.readdirSync(dir, { withFileTypes: true }).forEach(dirent => { + const fullPath = path.join(dir, dirent.name); + if (dirent.isDirectory()) { + emptyDirSync(fullPath); + fs.rmdirSync(fullPath); + } else { + fs.unlinkSync(fullPath); + } + }); +} + +export function rmRfSync(dir: string) { + emptyDirSync(dir); + fs.rmdirSync(dir); +} diff --git a/packages/cdk-bogus/lib/private/handlers/container-images.ts b/packages/cdk-bogus/lib/private/handlers/container-images.ts new file mode 100644 index 0000000000000..670c813dd8b20 --- /dev/null +++ b/packages/cdk-bogus/lib/private/handlers/container-images.ts @@ -0,0 +1,227 @@ +import * as path from 'path'; +import { DockerImageDestination } from '@aws-cdk/cloud-assembly-schema'; +import type * as AWS from 'aws-sdk'; +import { DockerImageManifestEntry } from '../../asset-manifest'; +import { EventType } from '../../progress'; +import { IAssetHandler, IHandlerHost } from '../asset-handler'; +import { Docker } from '../docker'; +import { replaceAwsPlaceholders } from '../placeholders'; +import { shell } from '../shell'; + +interface ContainerImageAssetHandlerInit { + readonly ecr: AWS.ECR; + readonly repoUri: string; + readonly imageUri: string; + readonly destinationAlreadyExists: boolean; +} + +export class ContainerImageAssetHandler implements IAssetHandler { + private init?: ContainerImageAssetHandlerInit; + + constructor( + private readonly workDir: string, + private readonly asset: DockerImageManifestEntry, + private readonly host: IHandlerHost) { + } + + public async build(): Promise { + const initOnce = await this.initOnce(); + + if (initOnce.destinationAlreadyExists) { return; } + if (this.host.aborted) { return; } + + const dockerForBuilding = await this.host.dockerFactory.forBuild({ + repoUri: initOnce.repoUri, + logger: (m: string) => this.host.emitMessage(EventType.DEBUG, m), + ecr: initOnce.ecr, + }); + + const builder = new ContainerImageBuilder(dockerForBuilding, this.workDir, this.asset, this.host); + const localTagName = await builder.build(); + + if (localTagName === undefined || this.host.aborted) { return; } + if (this.host.aborted) { return; } + + await dockerForBuilding.tag(localTagName, initOnce.imageUri); + } + + public async isPublished(): Promise { + try { + const initOnce = await this.initOnce({ quiet: true }); + return initOnce.destinationAlreadyExists; + } catch (e: any) { + this.host.emitMessage(EventType.DEBUG, `${e.message}`); + } + return false; + } + + public async publish(): Promise { + const initOnce = await this.initOnce(); + + if (initOnce.destinationAlreadyExists) { return; } + if (this.host.aborted) { return; } + + const dockerForPushing = await this.host.dockerFactory.forEcrPush({ + repoUri: initOnce.repoUri, + logger: (m: string) => this.host.emitMessage(EventType.DEBUG, m), + ecr: initOnce.ecr, + }); + + if (this.host.aborted) { return; } + + this.host.emitMessage(EventType.UPLOAD, `Push ${initOnce.imageUri}`); + await dockerForPushing.push(initOnce.imageUri); + } + + private async initOnce(options: { quiet?: boolean } = {}): Promise { + if (this.init) { + return this.init; + } + + const destination = await replaceAwsPlaceholders(this.asset.destination, this.host.aws); + const ecr = await this.host.aws.ecrClient({ + ...destination, + quiet: options.quiet, + }); + const account = async () => (await this.host.aws.discoverCurrentAccount())?.accountId; + + const repoUri = await repositoryUri(ecr, destination.repositoryName); + if (!repoUri) { + throw new Error(`No ECR repository named '${destination.repositoryName}' in account ${await account()}. Is this account bootstrapped?`); + } + + const imageUri = `${repoUri}:${destination.imageTag}`; + + this.init = { + imageUri, + ecr, + repoUri, + destinationAlreadyExists: await this.destinationAlreadyExists(ecr, destination, imageUri), + }; + + return this.init; + } + + /** + * Check whether the image already exists in the ECR repo + * + * Use the fields from the destination to do the actual check. The imageUri + * should correspond to that, but is only used to print Docker image location + * for user benefit (the format is slightly different). + */ + private async destinationAlreadyExists(ecr: AWS.ECR, destination: DockerImageDestination, imageUri: string): Promise { + this.host.emitMessage(EventType.CHECK, `Check ${imageUri}`); + if (await imageExists(ecr, destination.repositoryName, destination.imageTag)) { + this.host.emitMessage(EventType.FOUND, `Found ${imageUri}`); + return true; + } + + return false; + } +} + +class ContainerImageBuilder { + constructor( + private readonly docker: Docker, + private readonly workDir: string, + private readonly asset: DockerImageManifestEntry, + private readonly host: IHandlerHost) { + } + + async build(): Promise { + return this.asset.source.executable + ? this.buildExternalAsset(this.asset.source.executable) + : this.buildDirectoryAsset(); + } + + /** + * Build a (local) Docker asset from a directory with a Dockerfile + * + * Tags under a deterministic, unique, local identifier wich will skip + * the build if it already exists. + */ + private async buildDirectoryAsset(): Promise { + const localTagName = `cdkasset-${this.asset.id.assetId.toLowerCase()}`; + + if (!(await this.isImageCached(localTagName))) { + if (this.host.aborted) { return undefined; } + + await this.buildImage(localTagName); + } + + return localTagName; + } + + /** + * Build a (local) Docker asset by running an external command + * + * External command is responsible for deduplicating the build if possible, + * and is expected to return the generated image identifier on stdout. + */ + private async buildExternalAsset(executable: string[], cwd?: string): Promise { + const assetPath = cwd ?? this.workDir; + + this.host.emitMessage(EventType.BUILD, `Building Docker image using command '${executable}'`); + if (this.host.aborted) { return undefined; } + + return (await shell(executable, { cwd: assetPath, quiet: true })).trim(); + } + + private async buildImage(localTagName: string): Promise { + const source = this.asset.source; + if (!source.directory) { + throw new Error(`'directory' is expected in the DockerImage asset source, got: ${JSON.stringify(source)}`); + } + + const fullPath = path.resolve(this.workDir, source.directory); + this.host.emitMessage(EventType.BUILD, `Building Docker image at ${fullPath}`); + + await this.docker.build({ + directory: fullPath, + tag: localTagName, + buildArgs: source.dockerBuildArgs, + buildSecrets: source.dockerBuildSecrets, + target: source.dockerBuildTarget, + file: source.dockerFile, + networkMode: source.networkMode, + platform: source.platform, + outputs: source.dockerOutputs, + cacheFrom: source.cacheFrom, + cacheTo: source.cacheTo, + }); + } + + private async isImageCached(localTagName: string): Promise { + if (await this.docker.exists(localTagName)) { + this.host.emitMessage(EventType.CACHED, `Cached ${localTagName}`); + return true; + } + + return false; + } +} + +async function imageExists(ecr: AWS.ECR, repositoryName: string, imageTag: string) { + try { + await ecr.describeImages({ repositoryName, imageIds: [{ imageTag }] }).promise(); + return true; + } catch (e: any) { + if (e.code !== 'ImageNotFoundException') { throw e; } + return false; + } +} + +/** + * Return the URI for the repository with the given name + * + * Returns undefined if the repository does not exist. + */ +async function repositoryUri(ecr: AWS.ECR, repositoryName: string): Promise { + try { + const response = await ecr.describeRepositories({ repositoryNames: [repositoryName] }).promise(); + return (response.repositories || [])[0]?.repositoryUri; + } catch (e: any) { + if (e.code !== 'RepositoryNotFoundException') { throw e; } + return undefined; + } +} diff --git a/packages/cdk-bogus/lib/private/handlers/files.ts b/packages/cdk-bogus/lib/private/handlers/files.ts new file mode 100644 index 0000000000000..f9a928dd69727 --- /dev/null +++ b/packages/cdk-bogus/lib/private/handlers/files.ts @@ -0,0 +1,292 @@ +import { createReadStream, promises as fs } from 'fs'; +import * as path from 'path'; +import { FileAssetPackaging, FileSource } from '@aws-cdk/cloud-assembly-schema'; +import * as mime from 'mime'; +import { FileManifestEntry } from '../../asset-manifest'; +import { EventType } from '../../progress'; +import { zipDirectory } from '../archive'; +import { IAssetHandler, IHandlerHost } from '../asset-handler'; +import { pathExists } from '../fs-extra'; +import { replaceAwsPlaceholders } from '../placeholders'; +import { shell } from '../shell'; + +/** + * The size of an empty zip file is 22 bytes + * + * Ref: https://en.wikipedia.org/wiki/ZIP_(file_format) + */ +const EMPTY_ZIP_FILE_SIZE = 22; + +export class FileAssetHandler implements IAssetHandler { + private readonly fileCacheRoot: string; + + constructor( + private readonly workDir: string, + private readonly asset: FileManifestEntry, + private readonly host: IHandlerHost) { + this.fileCacheRoot = path.join(workDir, '.cache'); + } + + public async build(): Promise {} + + public async isPublished(): Promise { + const destination = await replaceAwsPlaceholders(this.asset.destination, this.host.aws); + const s3Url = `s3://${destination.bucketName}/${destination.objectKey}`; + try { + const s3 = await this.host.aws.s3Client({ + ...destination, + quiet: true, + }); + this.host.emitMessage(EventType.CHECK, `Check ${s3Url}`); + + if (await objectExists(s3, destination.bucketName, destination.objectKey)) { + this.host.emitMessage(EventType.FOUND, `Found ${s3Url}`); + return true; + } + } catch (e: any) { + this.host.emitMessage(EventType.DEBUG, `${e.message}`); + } + return false; + } + + public async publish(): Promise { + const destination = await replaceAwsPlaceholders(this.asset.destination, this.host.aws); + const s3Url = `s3://${destination.bucketName}/${destination.objectKey}`; + const s3 = await this.host.aws.s3Client(destination); + this.host.emitMessage(EventType.CHECK, `Check ${s3Url}`); + + const bucketInfo = BucketInformation.for(this.host); + + // A thunk for describing the current account. Used when we need to format an error + // message, not in the success case. + const account = async () => (await this.host.aws.discoverTargetAccount(destination))?.accountId; + switch (await bucketInfo.bucketOwnership(s3, destination.bucketName)) { + case BucketOwnership.MINE: + break; + case BucketOwnership.DOES_NOT_EXIST: + throw new Error(`No bucket named '${destination.bucketName}'. Is account ${await account()} bootstrapped?`); + case BucketOwnership.SOMEONE_ELSES_OR_NO_ACCESS: + throw new Error(`Bucket named '${destination.bucketName}' exists, but not in account ${await account()}. Wrong account?`); + } + + if (await objectExists(s3, destination.bucketName, destination.objectKey)) { + this.host.emitMessage(EventType.FOUND, `Found ${s3Url}`); + return; + } + + // Identify the the bucket encryption type to set the header on upload + // required for SCP rules denying uploads without encryption header + let paramsEncryption: {[index: string]:any}= {}; + const encryption2 = await bucketInfo.bucketEncryption(s3, destination.bucketName); + switch (encryption2.type) { + case 'no_encryption': + break; + case 'aes256': + paramsEncryption = { ServerSideEncryption: 'AES256' }; + break; + case 'kms': + // We must include the key ID otherwise S3 will encrypt with the default key + paramsEncryption = { + ServerSideEncryption: 'aws:kms', + SSEKMSKeyId: encryption2.kmsKeyId, + }; + break; + case 'does_not_exist': + this.host.emitMessage(EventType.DEBUG, `No bucket named '${destination.bucketName}'. Is account ${await account()} bootstrapped?`); + break; + case 'access_denied': + this.host.emitMessage(EventType.DEBUG, `Could not read encryption settings of bucket '${destination.bucketName}': uploading with default settings ("cdk bootstrap" to version 9 if your organization's policies prevent a successful upload or to get rid of this message).`); + break; + } + + if (this.host.aborted) { return; } + const publishFile = this.asset.source.executable ? + await this.externalPackageFile(this.asset.source.executable) : await this.packageFile(this.asset.source); + + this.host.emitMessage(EventType.UPLOAD, `Upload ${s3Url}`); + + const params = Object.assign({}, { + Bucket: destination.bucketName, + Key: destination.objectKey, + Body: createReadStream(publishFile.packagedPath), + ContentType: publishFile.contentType, + }, + paramsEncryption); + + await s3.upload(params).promise(); + } + + private async packageFile(source: FileSource): Promise { + if (!source.path) { + throw new Error(`'path' is expected in the File asset source, got: ${JSON.stringify(source)}`); + } + + const fullPath = path.resolve(this.workDir, source.path); + + if (source.packaging === FileAssetPackaging.ZIP_DIRECTORY) { + const contentType = 'application/zip'; + + await fs.mkdir(this.fileCacheRoot, { recursive: true }); + const packagedPath = path.join(this.fileCacheRoot, `${this.asset.id.assetId}.zip`); + + if (await pathExists(packagedPath)) { + this.host.emitMessage(EventType.CACHED, `From cache ${packagedPath}`); + return { packagedPath, contentType }; + } + + this.host.emitMessage(EventType.BUILD, `Zip ${fullPath} -> ${packagedPath}`); + await zipDirectory(fullPath, packagedPath, (m) => this.host.emitMessage(EventType.DEBUG, m)); + return { packagedPath, contentType }; + } else { + const contentType = mime.getType(fullPath) ?? 'application/octet-stream'; + return { packagedPath: fullPath, contentType }; + } + } + + private async externalPackageFile(executable: string[]): Promise { + this.host.emitMessage(EventType.BUILD, `Building asset source using command: '${executable}'`); + + return { + packagedPath: (await shell(executable, { quiet: true })).trim(), + contentType: 'application/zip', + }; + } +} + +enum BucketOwnership { + DOES_NOT_EXIST, + MINE, + SOMEONE_ELSES_OR_NO_ACCESS +} + +type BucketEncryption = + | { readonly type: 'no_encryption' } + | { readonly type: 'aes256' } + | { readonly type: 'kms'; readonly kmsKeyId?: string } + | { readonly type: 'access_denied' } + | { readonly type: 'does_not_exist' } + ; + +async function objectExists(s3: AWS.S3, bucket: string, key: string) { + /* + * The object existence check here refrains from using the `headObject` operation because this + * would create a negative cache entry, making GET-after-PUT eventually consistent. This has been + * observed to result in CloudFormation issuing "ValidationError: S3 error: Access Denied", for + * example in https://github.com/aws/aws-cdk/issues/6430. + * + * To prevent this, we are instead using the listObjectsV2 call, using the looked up key as the + * prefix, and limiting results to 1. Since the list operation returns keys ordered by binary + * UTF-8 representation, the key we are looking for is guaranteed to always be the first match + * returned if it exists. + * + * If the file is too small, we discount it as a cache hit. There is an issue + * somewhere that sometimes produces empty zip files, and we would otherwise + * never retry building those assets without users having to manually clear + * their bucket, which is a bad experience. + */ + const response = await s3.listObjectsV2({ Bucket: bucket, Prefix: key, MaxKeys: 1 }).promise(); + return ( + response.Contents != null && + response.Contents.some( + (object) => object.Key === key && (object.Size == null || object.Size > EMPTY_ZIP_FILE_SIZE), + ) + ); +} + +/** + * A packaged asset which can be uploaded (either a single file or directory) + */ +interface PackagedFileAsset { + /** + * Path of the file or directory + */ + readonly packagedPath: string; + + /** + * Content type to be added in the S3 upload action + * + * @default - No content type + */ + readonly contentType?: string; +} + +/** + * Cache for bucket information, so we don't have to keep doing the same calls again and again + * + * We scope the lifetime of the cache to the lifetime of the host, so that we don't have to do + * anything special for tests and yet the cache will live for the entire lifetime of the asset + * upload session when used by the CLI. + */ +class BucketInformation { + public static for(host: IHandlerHost) { + const existing = BucketInformation.caches.get(host); + if (existing) { return existing; } + + const fresh = new BucketInformation(); + BucketInformation.caches.set(host, fresh); + return fresh; + } + + private static readonly caches = new WeakMap(); + + private readonly ownerships = new Map(); + private readonly encryptions = new Map(); + + private constructor() { + } + + public async bucketOwnership(s3: AWS.S3, bucket: string): Promise { + return cached(this.ownerships, bucket, () => this._bucketOwnership(s3, bucket)); + } + + public async bucketEncryption(s3: AWS.S3, bucket: string): Promise { + return cached(this.encryptions, bucket, () => this._bucketEncryption(s3, bucket)); + } + + private async _bucketOwnership(s3: AWS.S3, bucket: string): Promise { + try { + await s3.getBucketLocation({ Bucket: bucket }).promise(); + return BucketOwnership.MINE; + } catch (e: any) { + if (e.code === 'NoSuchBucket') { return BucketOwnership.DOES_NOT_EXIST; } + if (['AccessDenied', 'AllAccessDisabled'].includes(e.code)) { return BucketOwnership.SOMEONE_ELSES_OR_NO_ACCESS; } + throw e; + } + } + + private async _bucketEncryption(s3: AWS.S3, bucket: string): Promise { + try { + const encryption = await s3.getBucketEncryption({ Bucket: bucket }).promise(); + const l = encryption?.ServerSideEncryptionConfiguration?.Rules?.length ?? 0; + if (l > 0) { + const apply = encryption?.ServerSideEncryptionConfiguration?.Rules[0]?.ApplyServerSideEncryptionByDefault; + let ssealgo = apply?.SSEAlgorithm; + if (ssealgo === 'AES256') return { type: 'aes256' }; + if (ssealgo === 'aws:kms') return { type: 'kms', kmsKeyId: apply?.KMSMasterKeyID }; + } + return { type: 'no_encryption' }; + } catch (e: any) { + if (e.code === 'NoSuchBucket') { + return { type: 'does_not_exist' }; + } + if (e.code === 'ServerSideEncryptionConfigurationNotFoundError') { + return { type: 'no_encryption' }; + } + + if (['AccessDenied', 'AllAccessDisabled'].includes(e.code)) { + return { type: 'access_denied' }; + } + return { type: 'no_encryption' }; + } + } +} + +async function cached(cache: Map, key: A, factory: (x: A) => Promise): Promise { + if (cache.has(key)) { + return cache.get(key)!; + } + + const fresh = await factory(key); + cache.set(key, fresh); + return fresh; +} diff --git a/packages/cdk-bogus/lib/private/handlers/index.ts b/packages/cdk-bogus/lib/private/handlers/index.ts new file mode 100644 index 0000000000000..1a46247674468 --- /dev/null +++ b/packages/cdk-bogus/lib/private/handlers/index.ts @@ -0,0 +1,15 @@ +import { ContainerImageAssetHandler } from './container-images'; +import { FileAssetHandler } from './files'; +import { AssetManifest, DockerImageManifestEntry, FileManifestEntry, IManifestEntry } from '../../asset-manifest'; +import { IAssetHandler, IHandlerHost } from '../asset-handler'; + +export function makeAssetHandler(manifest: AssetManifest, asset: IManifestEntry, host: IHandlerHost): IAssetHandler { + if (asset instanceof FileManifestEntry) { + return new FileAssetHandler(manifest.directory, asset, host); + } + if (asset instanceof DockerImageManifestEntry) { + return new ContainerImageAssetHandler(manifest.directory, asset, host); + } + + throw new Error(`Unrecognized asset type: '${asset}'`); +} diff --git a/packages/cdk-bogus/lib/private/placeholders.ts b/packages/cdk-bogus/lib/private/placeholders.ts new file mode 100644 index 0000000000000..50f76dfd3a7a6 --- /dev/null +++ b/packages/cdk-bogus/lib/private/placeholders.ts @@ -0,0 +1,34 @@ +import { EnvironmentPlaceholders } from '@aws-cdk/cx-api'; +import { IAws } from '../aws'; + +/** + * Replace the {ACCOUNT} and {REGION} placeholders in all strings found in a complex object. + * + * Duplicated between cdk-assets and aws-cdk CLI because we don't have a good single place to put it + * (they're nominally independent tools). + */ +export async function replaceAwsPlaceholders(object: A, aws: IAws): Promise { + let partition = async () => { + const p = await aws.discoverPartition(); + partition = () => Promise.resolve(p); + return p; + }; + + let account = async () => { + const a = await aws.discoverCurrentAccount(); + account = () => Promise.resolve(a); + return a; + }; + + return EnvironmentPlaceholders.replaceAsync(object, { + async region() { + return object.region ?? aws.discoverDefaultRegion(); + }, + async accountId() { + return (await account()).accountId; + }, + async partition() { + return partition(); + }, + }); +} \ No newline at end of file diff --git a/packages/cdk-bogus/lib/private/shell.ts b/packages/cdk-bogus/lib/private/shell.ts new file mode 100644 index 0000000000000..ba7837f49810e --- /dev/null +++ b/packages/cdk-bogus/lib/private/shell.ts @@ -0,0 +1,127 @@ +import * as child_process from 'child_process'; + +export type Logger = (x: string) => void; + +export interface ShellOptions extends child_process.SpawnOptions { + readonly quiet?: boolean; + readonly logger?: Logger; + readonly input?: string; +} + +/** + * OS helpers + * + * Shell function which both prints to stdout and collects the output into a + * string. + */ +export async function shell(command: string[], options: ShellOptions = {}): Promise { + if (options.logger) { + options.logger(renderCommandLine(command)); + } + const child = child_process.spawn(command[0], command.slice(1), { + ...options, + stdio: [options.input ? 'pipe' : 'ignore', 'pipe', 'pipe'], + }); + + return new Promise((resolve, reject) => { + if (options.input) { + child.stdin!.write(options.input); + child.stdin!.end(); + } + + const stdout = new Array(); + const stderr = new Array(); + + // Both write to stdout and collect + child.stdout!.on('data', chunk => { + if (!options.quiet) { + process.stdout.write(chunk); + } + stdout.push(chunk); + }); + + child.stderr!.on('data', chunk => { + if (!options.quiet) { + process.stderr.write(chunk); + } + + stderr.push(chunk); + }); + + child.once('error', reject); + + child.once('close', (code, signal) => { + if (code === 0) { + resolve(Buffer.concat(stdout).toString('utf-8')); + } else { + const out = Buffer.concat(stderr).toString('utf-8').trim(); + reject(new ProcessFailed(code, signal, `${renderCommandLine(command)} exited with ${code != null ? 'error code' : 'signal'} ${code ?? signal}: ${out}`)); + } + }); + }); +} + +export type ProcessFailedError = ProcessFailed + +class ProcessFailed extends Error { + public readonly code = 'PROCESS_FAILED'; + + constructor(public readonly exitCode: number | null, public readonly signal: NodeJS.Signals | null, message: string) { + super(message); + } +} + +/** + * Render the given command line as a string + * + * Probably missing some cases but giving it a good effort. + */ +function renderCommandLine(cmd: string[]) { + if (process.platform !== 'win32') { + return doRender(cmd, hasAnyChars(' ', '\\', '!', '"', "'", '&', '$'), posixEscape); + } else { + return doRender(cmd, hasAnyChars(' ', '"', '&', '^', '%'), windowsEscape); + } +} + +/** + * Render a UNIX command line + */ +function doRender(cmd: string[], needsEscaping: (x: string) => boolean, doEscape: (x: string) => string): string { + return cmd.map(x => needsEscaping(x) ? doEscape(x) : x).join(' '); +} + +/** + * Return a predicate that checks if a string has any of the indicated chars in it + */ +function hasAnyChars(...chars: string[]): (x: string) => boolean { + return (str: string) => { + return chars.some(c => str.indexOf(c) !== -1); + }; +} + +/** + * Escape a shell argument for POSIX shells + * + * Wrapping in single quotes and escaping single quotes inside will do it for us. + */ +function posixEscape(x: string) { + // Turn ' -> '"'"' + x = x.replace("'", "'\"'\"'"); + return `'${x}'`; +} + +/** + * Escape a shell argument for cmd.exe + * + * This is how to do it right, but I'm not following everything: + * + * https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/ + */ +function windowsEscape(x: string): string { + // First surround by double quotes, ignore the part about backslashes + x = `"${x}"`; + // Now escape all special characters + const shellMeta = new Set(['"', '&', '^', '%']); + return x.split('').map(c => shellMeta.has(x) ? '^' + c : c).join(''); +} diff --git a/packages/cdk-bogus/lib/private/util.ts b/packages/cdk-bogus/lib/private/util.ts new file mode 100644 index 0000000000000..88a87a18e6ba9 --- /dev/null +++ b/packages/cdk-bogus/lib/private/util.ts @@ -0,0 +1,12 @@ +/** + * Creates a critical section, ensuring that at most one function can + * enter the critical section at a time. + */ +export function createCriticalSection() { + let lock = Promise.resolve(); + return async (criticalFunction: () => Promise) => { + const res = lock.then(() => criticalFunction()); + lock = res.catch(e => e); + return res; + }; +}; \ No newline at end of file diff --git a/packages/cdk-bogus/lib/progress.ts b/packages/cdk-bogus/lib/progress.ts new file mode 100644 index 0000000000000..b2c8e77ddad78 --- /dev/null +++ b/packages/cdk-bogus/lib/progress.ts @@ -0,0 +1,86 @@ +import { IManifestEntry } from './asset-manifest'; + +/** + * A listener for progress events from the publisher + */ +export interface IPublishProgressListener { + /** + * Asset build event + */ + onPublishEvent(type: EventType, event: IPublishProgress): void; +} + +/** + * A single event for an asset + */ +export enum EventType { + /** + * Just starting on an asset + */ + START = 'start', + + /** + * When an asset is successfully finished + */ + SUCCESS = 'success', + + /** + * When an asset failed + */ + FAIL = 'fail', + + /** + * Checking whether an asset has already been published + */ + CHECK = 'check', + + /** + * The asset was already published + */ + FOUND = 'found', + + /** + * The asset was reused locally from a cached version + */ + CACHED = 'cached', + + /** + * The asset will be built + */ + BUILD = 'build', + + /** + * The asset will be uploaded + */ + UPLOAD = 'upload', + + /** + * Another type of detail message + */ + DEBUG = 'debug', +} + +/** + * Context object for publishing progress + */ +export interface IPublishProgress { + /** + * Current event message + */ + readonly message: string; + + /** + * Asset currently being packaged (if any) + */ + readonly currentAsset?: IManifestEntry; + + /** + * How far along are we? + */ + readonly percentComplete: number; + + /** + * Abort the current publishing operation + */ + abort(): void; +} diff --git a/packages/cdk-bogus/lib/publishing.ts b/packages/cdk-bogus/lib/publishing.ts new file mode 100644 index 0000000000000..9e38308cd7e66 --- /dev/null +++ b/packages/cdk-bogus/lib/publishing.ts @@ -0,0 +1,247 @@ +import { AssetManifest, IManifestEntry } from './asset-manifest'; +import { IAws } from './aws'; +import { IAssetHandler, IHandlerHost } from './private/asset-handler'; +import { DockerFactory } from './private/docker'; +import { makeAssetHandler } from './private/handlers'; +import { EventType, IPublishProgress, IPublishProgressListener } from './progress'; + +export interface AssetPublishingOptions { + /** + * Entry point for AWS client + */ + readonly aws: IAws; + + /** + * Listener for progress events + * + * @default No listener + */ + readonly progressListener?: IPublishProgressListener; + + /** + * Whether to throw at the end if there were errors + * + * @default true + */ + readonly throwOnError?: boolean; + + /** + * Whether to publish in parallel, when 'publish()' is called + * + * @default false + */ + readonly publishInParallel?: boolean; + + /** + * Whether to build assets, when 'publish()' is called + * + * @default true + */ + readonly buildAssets?: boolean; + + /** + * Whether to publish assets, when 'publish()' is called + * + * @default true + */ + readonly publishAssets?: boolean; +} + +/** + * A failure to publish an asset + */ +export interface FailedAsset { + /** + * The asset that failed to publish + */ + readonly asset: IManifestEntry; + + /** + * The failure that occurred + */ + readonly error: Error; +} + +export class AssetPublishing implements IPublishProgress { + /** + * The message for the IPublishProgress interface + */ + public message: string = 'Starting'; + + /** + * The current asset for the IPublishProgress interface + */ + public currentAsset?: IManifestEntry; + public readonly failures = new Array(); + private readonly assets: IManifestEntry[]; + + private readonly totalOperations: number; + private completedOperations: number = 0; + private aborted = false; + private readonly handlerHost: IHandlerHost; + private readonly publishInParallel: boolean; + private readonly buildAssets: boolean; + private readonly publishAssets: boolean; + private readonly handlerCache = new Map(); + + constructor(private readonly manifest: AssetManifest, private readonly options: AssetPublishingOptions) { + this.assets = manifest.entries; + this.totalOperations = this.assets.length; + this.publishInParallel = options.publishInParallel ?? false; + this.buildAssets = options.buildAssets ?? true; + this.publishAssets = options.publishAssets ?? true; + + const self = this; + this.handlerHost = { + aws: this.options.aws, + get aborted() { return self.aborted; }, + emitMessage(t, m) { self.progressEvent(t, m); }, + dockerFactory: new DockerFactory(), + }; + } + + /** + * Publish all assets from the manifest + */ + public async publish(): Promise { + if (this.publishInParallel) { + await Promise.all(this.assets.map(async (asset) => this.publishAsset(asset))); + } else { + for (const asset of this.assets) { + if (!await this.publishAsset(asset)) { + break; + } + } + } + + if ((this.options.throwOnError ?? true) && this.failures.length > 0) { + throw new Error(`Error publishing: ${this.failures.map(e => e.error.message)}`); + } + } + + /** + * Build a single asset from the manifest + */ + public async buildEntry(asset: IManifestEntry) { + try { + if (this.progressEvent(EventType.START, `Building ${asset.id}`)) { return false; } + + const handler = this.assetHandler(asset); + await handler.build(); + + if (this.aborted) { + throw new Error('Aborted'); + } + + this.completedOperations++; + if (this.progressEvent(EventType.SUCCESS, `Built ${asset.id}`)) { return false; } + } catch (e: any) { + this.failures.push({ asset, error: e }); + this.completedOperations++; + if (this.progressEvent(EventType.FAIL, e.message)) { return false; } + } + + return true; + } + + /** + * Publish a single asset from the manifest + */ + public async publishEntry(asset: IManifestEntry) { + try { + if (this.progressEvent(EventType.START, `Publishing ${asset.id}`)) { return false; } + + const handler = this.assetHandler(asset); + await handler.publish(); + + if (this.aborted) { + throw new Error('Aborted'); + } + + this.completedOperations++; + if (this.progressEvent(EventType.SUCCESS, `Published ${asset.id}`)) { return false; } + } catch (e: any) { + this.failures.push({ asset, error: e }); + this.completedOperations++; + if (this.progressEvent(EventType.FAIL, e.message)) { return false; } + } + + return true; + } + + /** + * Return whether a single asset is published + */ + public isEntryPublished(asset: IManifestEntry) { + const handler = this.assetHandler(asset); + return handler.isPublished(); + } + + /** + * publish an asset (used by 'publish()') + * @param asset The asset to publish + * @returns false when publishing should stop + */ + private async publishAsset(asset: IManifestEntry) { + try { + if (this.progressEvent(EventType.START, `Publishing ${asset.id}`)) { return false; } + + const handler = this.assetHandler(asset); + + if (this.buildAssets) { + await handler.build(); + } + + if (this.publishAssets) { + await handler.publish(); + } + + if (this.aborted) { + throw new Error('Aborted'); + } + + this.completedOperations++; + if (this.progressEvent(EventType.SUCCESS, `Published ${asset.id}`)) { return false; } + } catch (e: any) { + this.failures.push({ asset, error: e }); + this.completedOperations++; + if (this.progressEvent(EventType.FAIL, e.message)) { return false; } + } + + return true; + } + + public get percentComplete() { + if (this.totalOperations === 0) { return 100; } + return Math.floor((this.completedOperations / this.totalOperations) * 100); + } + + public abort(): void { + this.aborted = true; + } + + public get hasFailures() { + return this.failures.length > 0; + } + + /** + * Publish a progress event to the listener, if present. + * + * Returns whether an abort is requested. Helper to get rid of repetitive code in publish(). + */ + private progressEvent(event: EventType, message: string): boolean { + this.message = message; + if (this.options.progressListener) { this.options.progressListener.onPublishEvent(event, this); } + return this.aborted; + } + + private assetHandler(asset: IManifestEntry) { + const existing = this.handlerCache.get(asset); + if (existing) { + return existing; + } + const ret = makeAssetHandler(this.manifest, asset, this.handlerHost); + this.handlerCache.set(asset, ret); + return ret; + } +} diff --git a/packages/cdk-bogus/package.json b/packages/cdk-bogus/package.json new file mode 100644 index 0000000000000..9ac5859db6b2f --- /dev/null +++ b/packages/cdk-bogus/package.json @@ -0,0 +1,82 @@ +{ + "name": "cdk-bogus", + "description": "CDK Asset Publishing Tool", + "version": "0.0.0", + "main": "lib/index.js", + "types": "lib/index.d.ts", + "bin": { + "cdk-assets": "bin/cdk-assets", + "docker-credential-cdk-assets": "bin/docker-credential-cdk-assets" + }, + "scripts": { + "build": "cdk-build", + "integ": "integ-runner", + "lint": "cdk-lint", + "package": "cdk-package", + "awslint": "cdk-awslint", + "pkglint": "pkglint -f", + "test": "cdk-test", + "watch": "cdk-watch", + "build+test": "yarn build && yarn test", + "build+test+package": "yarn build+test && yarn package", + "compat": "cdk-compat", + "build+extract": "yarn build", + "build+test+extract": "yarn build+test" + }, + "author": { + "name": "Amazon Web Services", + "url": "https://aws.amazon.com", + "organization": true + }, + "license": "Apache-2.0", + "devDependencies": { + "@types/archiver": "^5.3.2", + "@types/glob": "^7.2.0", + "@types/jest": "^29.5.1", + "@types/mime": "^2.0.3", + "@types/mock-fs": "^4.13.1", + "@types/yargs": "^15.0.15", + "@aws-cdk/cdk-build-tools": "0.0.0", + "jest": "^29.5.0", + "jszip": "^3.10.1", + "mock-fs": "^4.14.0", + "@aws-cdk/pkglint": "0.0.0" + }, + "dependencies": { + "@aws-cdk/cloud-assembly-schema": "0.0.0", + "@aws-cdk/cx-api": "0.0.0", + "archiver": "^5.3.1", + "aws-sdk": "^2.1379.0", + "glob": "^7.2.3", + "mime": "^2.6.0", + "yargs": "^16.2.0" + }, + "repository": { + "url": "https://github.com/aws/aws-cdk.git", + "type": "git", + "directory": "packages/cdk-assets" + }, + "keywords": [ + "aws", + "cdk" + ], + "homepage": "https://github.com/aws/aws-cdk", + "engines": { + "node": ">= 14.15.0" + }, + "cdk-package": { + "shrinkWrap": true + }, + "nozem": { + "ostools": [ + "unzip", + "diff", + "rm" + ] + }, + "stability": "stable", + "maturity": "stable", + "publishConfig": { + "tag": "latest" + } +} diff --git a/packages/cdk-bogus/test/archive.test.ts b/packages/cdk-bogus/test/archive.test.ts new file mode 100644 index 0000000000000..d0fe1e2b3dbe7 --- /dev/null +++ b/packages/cdk-bogus/test/archive.test.ts @@ -0,0 +1,94 @@ +import { exec as _exec } from 'child_process'; +import * as crypto from 'crypto'; +import { constants, exists, promises as fs } from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { promisify } from 'util'; +import * as jszip from 'jszip'; +import { zipDirectory } from '../lib/private/archive'; +import { rmRfSync } from '../lib/private/fs-extra'; +const exec = promisify(_exec); +const pathExists = promisify(exists); + +function logger(x: string) { + // eslint-disable-next-line no-console + console.log(x); +} + +test('zipDirectory can take a directory and produce a zip from it', async () => { + const stagingDir = await fs.mkdtemp(path.join(os.tmpdir(), 'test.archive')); + const extractDir = await fs.mkdtemp(path.join(os.tmpdir(), 'test.archive.extract')); + try { + const zipFile = path.join(stagingDir, 'output.zip'); + const originalDir = path.join(__dirname, 'test-archive'); + await zipDirectory(originalDir, zipFile, logger); + + // unzip and verify that the resulting tree is the same + await exec(`unzip ${zipFile}`, { cwd: extractDir }); + + await expect(exec(`diff -bur ${originalDir} ${extractDir}`)).resolves.toBeTruthy(); + + // inspect the zip file to check that dates are reset + const zip = await fs.readFile(zipFile); + const zipData = await jszip.loadAsync(zip); + const dates = Object.values(zipData.files).map(file => file.date.toISOString()); + expect(dates[0]).toBe('1980-01-01T00:00:00.000Z'); + expect(new Set(dates).size).toBe(1); + + // check that mode is preserved + const stat = await fs.stat(path.join(extractDir, 'executable.txt')); + // eslint-disable-next-line no-bitwise + const isExec = (stat.mode & constants.S_IXUSR) || (stat.mode & constants.S_IXGRP) || (stat.mode & constants.S_IXOTH); + expect(isExec).toBeTruthy(); + } finally { + rmRfSync(stagingDir); + rmRfSync(extractDir); + } +}); + +test('md5 hash of a zip stays consistent across invocations', async () => { + const stagingDir = await fs.mkdtemp(path.join(os.tmpdir(), 'test.archive')); + const zipFile1 = path.join(stagingDir, 'output.zip'); + const zipFile2 = path.join(stagingDir, 'output.zip'); + const originalDir = path.join(__dirname, 'test-archive'); + await zipDirectory(originalDir, zipFile1, logger); + await new Promise(ok => setTimeout(ok, 2000)); // wait 2s + await zipDirectory(originalDir, zipFile2, logger); + + const hash1 = contentHash(await fs.readFile(zipFile1)); + const hash2 = contentHash(await fs.readFile(zipFile2)); + + expect(hash1).toEqual(hash2); +}); + +test('zipDirectory follows symlinks', async () => { + const stagingDir = await fs.mkdtemp(path.join(os.tmpdir(), 'test.archive')); + const extractDir = await fs.mkdtemp(path.join(os.tmpdir(), 'test.archive.follow')); + try { + // First MAKE the symlink we're going to follow. We can't check it into git, because + // CodeBuild/CodePipeline (I forget which) is going to replace symlinks with a textual + // representation of its target upon checkout, for security reasons. So, to make sure + // the symlink exists, we need to create it at build time. + const symlinkPath = path.join(__dirname, 'test-archive-follow', 'data', 'linked'); + const symlinkTarget = '../linked'; + + if (await pathExists(symlinkPath)) { + await fs.unlink(symlinkPath); + } + await fs.symlink(symlinkTarget, symlinkPath, 'dir'); + + const originalDir = path.join(__dirname, 'test-archive-follow', 'data'); + const zipFile = path.join(stagingDir, 'output.zip'); + + await expect(zipDirectory(originalDir, zipFile, logger)).resolves.toBeUndefined(); + await expect(exec(`unzip ${zipFile}`, { cwd: extractDir })).resolves.toBeDefined(); + await expect(exec(`diff -bur ${originalDir} ${extractDir}`)).resolves.toBeDefined(); + } finally { + rmRfSync(stagingDir); + rmRfSync(extractDir); + } +}); + +function contentHash(data: string | Buffer | DataView) { + return crypto.createHash('sha256').update(data).digest('hex'); +} diff --git a/packages/cdk-bogus/test/bockfs.ts b/packages/cdk-bogus/test/bockfs.ts new file mode 100644 index 0000000000000..ffdc43aa9b6fa --- /dev/null +++ b/packages/cdk-bogus/test/bockfs.ts @@ -0,0 +1,32 @@ +// A not-so-fake filesystem mock similar to mock-fs +import * as fs from 'fs'; +import * as os from 'os'; +import * as path_ from 'path'; +import { rmRfSync } from '../lib/private/fs-extra'; + +const bockFsRoot = path_.join(os.tmpdir(), 'bockfs'); + +function bockfs(files: Record) { + for (const [fileName, contents] of Object.entries(files)) { + bockfs.write(fileName, contents); + } +} + +namespace bockfs { + export function write(fileName: string, contents: string) { + const fullPath = path(fileName); + fs.mkdirSync(path_.dirname(fullPath), { recursive: true }); + fs.writeFileSync(fullPath, contents, { encoding: 'utf-8' }); + } + + export function path(x: string) { + if (x.startsWith('/')) { x = x.slice(1); } // Force path to be non-absolute + return path_.join(bockFsRoot, x); + } + + export function restore() { + rmRfSync(bockFsRoot); + } +} + +export = bockfs; \ No newline at end of file diff --git a/packages/cdk-bogus/test/docker-images.test.ts b/packages/cdk-bogus/test/docker-images.test.ts new file mode 100644 index 0000000000000..561e37c823916 --- /dev/null +++ b/packages/cdk-bogus/test/docker-images.test.ts @@ -0,0 +1,663 @@ +jest.mock('child_process'); + +import * as fs from 'fs'; +import { Manifest } from '@aws-cdk/cloud-assembly-schema'; +import * as mockfs from 'mock-fs'; +import { mockAws, mockedApiFailure, mockedApiResult } from './mock-aws'; +import { mockSpawn } from './mock-child_process'; +import { AssetManifest, AssetPublishing } from '../lib'; +import * as dockercreds from '../lib/private/docker-credentials'; + +let aws: ReturnType; +const absoluteDockerPath = '/simple/cdk.out/dockerdir'; +beforeEach(() => { + jest.resetAllMocks(); + delete(process.env.CDK_DOCKER); + + // By default, assume no externally-configured credentials. + jest.spyOn(dockercreds, 'cdkCredentialsConfig').mockReturnValue(undefined); + + mockfs({ + '/simple/cdk.out/assets.json': JSON.stringify({ + version: Manifest.version(), + dockerImages: { + theAsset: { + source: { + directory: 'dockerdir', + }, + destinations: { + theDestination: { + region: 'us-north-50', + assumeRoleArn: 'arn:aws:role', + repositoryName: 'repo', + imageTag: 'abcdef', + }, + }, + }, + }, + }), + '/multi/cdk.out/assets.json': JSON.stringify({ + version: Manifest.version(), + dockerImages: { + theAsset1: { + source: { + directory: 'dockerdir', + }, + destinations: { + theDestination: { + region: 'us-north-50', + assumeRoleArn: 'arn:aws:role', + repositoryName: 'repo', + imageTag: 'theAsset1', + }, + }, + }, + theAsset2: { + source: { + directory: 'dockerdir', + }, + destinations: { + theDestination: { + region: 'us-north-50', + assumeRoleArn: 'arn:aws:role', + repositoryName: 'repo', + imageTag: 'theAsset2', + }, + }, + }, + }, + }), + '/external/cdk.out/assets.json': JSON.stringify({ + version: Manifest.version(), + dockerImages: { + theExternalAsset: { + source: { + executable: ['sometool'], + }, + destinations: { + theDestination: { + region: 'us-north-50', + assumeRoleArn: 'arn:aws:role', + repositoryName: 'repo', + imageTag: 'ghijkl', + }, + }, + }, + }, + }), + '/simple/cdk.out/dockerdir/Dockerfile': 'FROM scratch', + '/abs/cdk.out/assets.json': JSON.stringify({ + version: Manifest.version(), + dockerImages: { + theAsset: { + source: { + directory: absoluteDockerPath, + }, + destinations: { + theDestination: { + region: 'us-north-50', + assumeRoleArn: 'arn:aws:role', + repositoryName: 'repo', + imageTag: 'abcdef', + }, + }, + }, + }, + }), + '/default-network/cdk.out/assets.json': JSON.stringify({ + version: Manifest.version(), + dockerImages: { + theAsset: { + source: { + directory: 'dockerdir', + networkMode: 'default', + }, + destinations: { + theDestination: { + region: 'us-north-50', + assumeRoleArn: 'arn:aws:role', + repositoryName: 'repo', + imageTag: 'nopqr', + }, + }, + }, + }, + }), + '/default-network/cdk.out/dockerdir/Dockerfile': 'FROM scratch', + '/platform-arm64/cdk.out/assets.json': JSON.stringify({ + version: Manifest.version(), + dockerImages: { + theAsset: { + source: { + directory: 'dockerdir', + platform: 'linux/arm64', + }, + destinations: { + theDestination: { + region: 'us-north-50', + assumeRoleArn: 'arn:aws:role', + repositoryName: 'repo', + imageTag: 'nopqr', + }, + }, + }, + }, + }), + '/cache/cdk.out/assets.json': JSON.stringify({ + version: Manifest.version(), + dockerImages: { + theAsset: { + source: { + directory: 'dockerdir', + cacheFrom: [{ type: 'registry', params: { ref: 'abcdef' } }], + cacheTo: { type: 'inline' }, + }, + destinations: { + theDestination: { + region: 'us-north-50', + assumeRoleArn: 'arn:aws:role', + repositoryName: 'repo', + imageTag: 'nopqr', + }, + }, + }, + }, + }), + '/cache-from-multiple/cdk.out/assets.json': JSON.stringify({ + version: Manifest.version(), + dockerImages: { + theAsset: { + source: { + directory: 'dockerdir', + cacheFrom: [ + { type: 'registry', params: { ref: 'cache:ref' } }, + { type: 'registry', params: { ref: 'cache:main' } }, + { type: 'gha' }, + ], + }, + destinations: { + theDestination: { + region: 'us-north-50', + assumeRoleArn: 'arn:aws:role', + repositoryName: 'repo', + imageTag: 'nopqr', + }, + }, + }, + }, + }), + '/cache-to-complex/cdk.out/assets.json': JSON.stringify({ + version: Manifest.version(), + dockerImages: { + theAsset: { + source: { + directory: 'dockerdir', + cacheTo: { type: 'registry', params: { ref: 'cache:main', mode: 'max', compression: 'zstd' } }, + }, + destinations: { + theDestination: { + region: 'us-north-50', + assumeRoleArn: 'arn:aws:role', + repositoryName: 'repo', + imageTag: 'nopqr', + }, + }, + }, + }, + }), + '/platform-arm64/cdk.out/dockerdir/Dockerfile': 'FROM scratch', + }); + + aws = mockAws(); +}); + +afterEach(() => { + mockfs.restore(); +}); + +test('pass destination properties to AWS client', async () => { + const pub = new AssetPublishing(AssetManifest.fromPath('/simple/cdk.out'), { aws, throwOnError: false }); + + await pub.publish(); + + expect(aws.ecrClient).toHaveBeenCalledWith(expect.objectContaining({ + region: 'us-north-50', + assumeRoleArn: 'arn:aws:role', + })); +}); + +describe('with a complete manifest', () => { + let pub: AssetPublishing; + beforeEach(() => { + pub = new AssetPublishing(AssetManifest.fromPath('/simple/cdk.out'), { aws }); + }); + + test('Do nothing if docker image already exists', async () => { + aws.mockEcr.describeImages = mockedApiResult({ /* No error == image exists */ }); + + await pub.publish(); + + expect(aws.mockEcr.describeImages).toHaveBeenCalledWith(expect.objectContaining({ + imageIds: [{ imageTag: 'abcdef' }], + repositoryName: 'repo', + })); + }); + + test('Displays an error if the ECR repository cannot be found', async () => { + aws.mockEcr.describeImages = mockedApiFailure('RepositoryNotFoundException', 'Repository not Found'); + + await expect(pub.publish()).rejects.toThrow('Error publishing: Repository not Found'); + }); + + test('successful run does not need to query account ID', async () => { + aws.mockEcr.describeImages = mockedApiResult({ /* No error == image exists */ }); + await pub.publish(); + expect(aws.discoverCurrentAccount).not.toHaveBeenCalled(); + }); + + test('upload docker image if not uploaded yet but exists locally', async () => { + aws.mockEcr.describeImages = mockedApiFailure('ImageNotFoundException', 'File does not exist'); + aws.mockEcr.getAuthorizationToken = mockedApiResult({ + authorizationData: [ + { authorizationToken: 'dXNlcjpwYXNz', proxyEndpoint: 'https://proxy.com/' }, + ], + }); + + const expectAllSpawns = mockSpawn( + { commandLine: ['docker', 'login', '--username', 'user', '--password-stdin', 'https://proxy.com/'] }, + { commandLine: ['docker', 'inspect', 'cdkasset-theasset'] }, + { commandLine: ['docker', 'tag', 'cdkasset-theasset', '12345.amazonaws.com/repo:abcdef'] }, + { commandLine: ['docker', 'push', '12345.amazonaws.com/repo:abcdef'] }, + ); + + await pub.publish(); + + expectAllSpawns(); + expect(true).toBeTruthy(); // Expect no exception, satisfy linter + }); + + test('build and upload docker image if not exists anywhere', async () => { + aws.mockEcr.describeImages = mockedApiFailure('ImageNotFoundException', 'File does not exist'); + aws.mockEcr.getAuthorizationToken = mockedApiResult({ + authorizationData: [ + { authorizationToken: 'dXNlcjpwYXNz', proxyEndpoint: 'https://proxy.com/' }, + ], + }); + + const expectAllSpawns = mockSpawn( + { commandLine: ['docker', 'login', '--username', 'user', '--password-stdin', 'https://proxy.com/'] }, + { commandLine: ['docker', 'inspect', 'cdkasset-theasset'], exitCode: 1 }, + { commandLine: ['docker', 'build', '--tag', 'cdkasset-theasset', '.'], cwd: absoluteDockerPath }, + { commandLine: ['docker', 'tag', 'cdkasset-theasset', '12345.amazonaws.com/repo:abcdef'] }, + { commandLine: ['docker', 'push', '12345.amazonaws.com/repo:abcdef'] }, + ); + + await pub.publish(); + + expectAllSpawns(); + expect(true).toBeTruthy(); // Expect no exception, satisfy linter + }); + + test('build with networkMode option', async () => { + pub = new AssetPublishing(AssetManifest.fromPath('/default-network/cdk.out'), { aws }); + const defaultNetworkDockerpath = '/default-network/cdk.out/dockerdir'; + aws.mockEcr.describeImages = mockedApiFailure('ImageNotFoundException', 'File does not exist'); + aws.mockEcr.getAuthorizationToken = mockedApiResult({ + authorizationData: [ + { authorizationToken: 'dXNlcjpwYXNz', proxyEndpoint: 'https://proxy.com/' }, + ], + }); + + const expectAllSpawns = mockSpawn( + { commandLine: ['docker', 'login', '--username', 'user', '--password-stdin', 'https://proxy.com/'] }, + { commandLine: ['docker', 'inspect', 'cdkasset-theasset'], exitCode: 1 }, + { commandLine: ['docker', 'build', '--tag', 'cdkasset-theasset', '--network', 'default', '.'], cwd: defaultNetworkDockerpath }, + { commandLine: ['docker', 'tag', 'cdkasset-theasset', '12345.amazonaws.com/repo:nopqr'] }, + { commandLine: ['docker', 'push', '12345.amazonaws.com/repo:nopqr'] }, + ); + + await pub.publish(); + + expectAllSpawns(); + expect(true).toBeTruthy(); // Expect no exception, satisfy linter + }); + + test('build with platform option', async () => { + pub = new AssetPublishing(AssetManifest.fromPath('/platform-arm64/cdk.out'), { aws }); + const defaultNetworkDockerpath = '/platform-arm64/cdk.out/dockerdir'; + aws.mockEcr.describeImages = mockedApiFailure('ImageNotFoundException', 'File does not exist'); + aws.mockEcr.getAuthorizationToken = mockedApiResult({ + authorizationData: [ + { authorizationToken: 'dXNlcjpwYXNz', proxyEndpoint: 'https://proxy.com/' }, + ], + }); + + const expectAllSpawns = mockSpawn( + { commandLine: ['docker', 'login', '--username', 'user', '--password-stdin', 'https://proxy.com/'] }, + { commandLine: ['docker', 'inspect', 'cdkasset-theasset'], exitCode: 1 }, + { commandLine: ['docker', 'build', '--tag', 'cdkasset-theasset', '--platform', 'linux/arm64', '.'], cwd: defaultNetworkDockerpath }, + { commandLine: ['docker', 'tag', 'cdkasset-theasset', '12345.amazonaws.com/repo:nopqr'] }, + { commandLine: ['docker', 'push', '12345.amazonaws.com/repo:nopqr'] }, + ); + + await pub.publish(); + + expectAllSpawns(); + expect(true).toBeTruthy(); // Expect no exception, satisfy linter + }); + + test('build with cache option', async () => { + pub = new AssetPublishing(AssetManifest.fromPath('/cache/cdk.out'), { aws }); + const defaultNetworkDockerpath = '/cache/cdk.out/dockerdir'; + aws.mockEcr.describeImages = mockedApiFailure('ImageNotFoundException', 'File does not exist'); + aws.mockEcr.getAuthorizationToken = mockedApiResult({ + authorizationData: [ + { authorizationToken: 'dXNlcjpwYXNz', proxyEndpoint: 'https://proxy.com/' }, + ], + }); + + const expectAllSpawns = mockSpawn( + { commandLine: ['docker', 'login', '--username', 'user', '--password-stdin', 'https://proxy.com/'] }, + { commandLine: ['docker', 'inspect', 'cdkasset-theasset'], exitCode: 1 }, + { commandLine: ['docker', 'build', '--tag', 'cdkasset-theasset', '--cache-from', 'type=registry,ref=abcdef', '--cache-to', 'type=inline', '.'], cwd: defaultNetworkDockerpath }, + { commandLine: ['docker', 'tag', 'cdkasset-theasset', '12345.amazonaws.com/repo:nopqr'] }, + { commandLine: ['docker', 'push', '12345.amazonaws.com/repo:nopqr'] }, + ); + + await pub.publish(); + + expectAllSpawns(); + expect(true).toBeTruthy(); // Expect no exception, satisfy linter + }); + + test('build with multiple cache from option', async () => { + pub = new AssetPublishing(AssetManifest.fromPath('/cache-from-multiple/cdk.out'), { aws }); + const defaultNetworkDockerpath = '/cache-from-multiple/cdk.out/dockerdir'; + aws.mockEcr.describeImages = mockedApiFailure('ImageNotFoundException', 'File does not exist'); + aws.mockEcr.getAuthorizationToken = mockedApiResult({ + authorizationData: [ + { authorizationToken: 'dXNlcjpwYXNz', proxyEndpoint: 'https://proxy.com/' }, + ], + }); + + const expectAllSpawns = mockSpawn( + { commandLine: ['docker', 'login', '--username', 'user', '--password-stdin', 'https://proxy.com/'] }, + { commandLine: ['docker', 'inspect', 'cdkasset-theasset'], exitCode: 1 }, + { + commandLine: [ + 'docker', 'build', '--tag', 'cdkasset-theasset', '--cache-from', 'type=registry,ref=cache:ref', '--cache-from', 'type=registry,ref=cache:main', '--cache-from', 'type=gha', '.', + ], + cwd: defaultNetworkDockerpath, + }, + { commandLine: ['docker', 'tag', 'cdkasset-theasset', '12345.amazonaws.com/repo:nopqr'] }, + { commandLine: ['docker', 'push', '12345.amazonaws.com/repo:nopqr'] }, + ); + + await pub.publish(); + + expectAllSpawns(); + expect(true).toBeTruthy(); // Expect no exception, satisfy linter + }); + + test('build with cache to complex option', async () => { + pub = new AssetPublishing(AssetManifest.fromPath('/cache-to-complex/cdk.out'), { aws }); + const defaultNetworkDockerpath = '/cache-to-complex/cdk.out/dockerdir'; + aws.mockEcr.describeImages = mockedApiFailure('ImageNotFoundException', 'File does not exist'); + aws.mockEcr.getAuthorizationToken = mockedApiResult({ + authorizationData: [ + { authorizationToken: 'dXNlcjpwYXNz', proxyEndpoint: 'https://proxy.com/' }, + ], + }); + + const expectAllSpawns = mockSpawn( + { commandLine: ['docker', 'login', '--username', 'user', '--password-stdin', 'https://proxy.com/'] }, + { commandLine: ['docker', 'inspect', 'cdkasset-theasset'], exitCode: 1 }, + { commandLine: ['docker', 'build', '--tag', 'cdkasset-theasset', '--cache-to', 'type=registry,ref=cache:main,mode=max,compression=zstd', '.'], cwd: defaultNetworkDockerpath }, + { commandLine: ['docker', 'tag', 'cdkasset-theasset', '12345.amazonaws.com/repo:nopqr'] }, + { commandLine: ['docker', 'push', '12345.amazonaws.com/repo:nopqr'] }, + ); + + await pub.publish(); + + expectAllSpawns(); + expect(true).toBeTruthy(); // Expect no exception, satisfy linter + }); +}); + +describe('external assets', () => { + let pub: AssetPublishing; + const externalTag = 'external:tag'; + beforeEach(() => { + pub = new AssetPublishing(AssetManifest.fromPath('/external/cdk.out'), { aws }); + }); + + test('upload externally generated Docker image', async () => { + aws.mockEcr.describeImages = mockedApiFailure('ImageNotFoundException', 'File does not exist'); + aws.mockEcr.getAuthorizationToken = mockedApiResult({ + authorizationData: [ + { authorizationToken: 'dXNlcjpwYXNz', proxyEndpoint: 'https://proxy.com/' }, + ], + }); + + const expectAllSpawns = mockSpawn( + { commandLine: ['docker', 'login', '--username', 'user', '--password-stdin', 'https://proxy.com/'] }, + { commandLine: ['sometool'], stdout: externalTag, cwd: '/external/cdk.out' }, + { commandLine: ['docker', 'tag', externalTag, '12345.amazonaws.com/repo:ghijkl'] }, + { commandLine: ['docker', 'push', '12345.amazonaws.com/repo:ghijkl'] }, + ); + + await pub.publish(); + + expect(aws.ecrClient).toHaveBeenCalledWith(expect.objectContaining({ + region: 'us-north-50', + assumeRoleArn: 'arn:aws:role', + })); + expectAllSpawns(); + }); +}); + +test('correctly identify Docker directory if path is absolute', async () => { + const pub = new AssetPublishing(AssetManifest.fromPath('/abs/cdk.out'), { aws }); + + aws.mockEcr.describeImages = mockedApiFailure('ImageNotFoundException', 'File does not exist'); + aws.mockEcr.getAuthorizationToken = mockedApiResult({ + authorizationData: [ + { authorizationToken: 'dXNlcjpwYXNz', proxyEndpoint: 'https://proxy.com/' }, + ], + }); + + const expectAllSpawns = mockSpawn( + // Only care about the 'build' command line + { commandLine: ['docker', 'login'], prefix: true }, + { commandLine: ['docker', 'inspect'], exitCode: 1, prefix: true }, + { commandLine: ['docker', 'build', '--tag', 'cdkasset-theasset', '.'], cwd: absoluteDockerPath }, + { commandLine: ['docker', 'tag'], prefix: true }, + { commandLine: ['docker', 'push'], prefix: true }, + ); + + await pub.publish(); + + expect(true).toBeTruthy(); // Expect no exception, satisfy linter + expectAllSpawns(); +}); + +test('when external credentials are present, explicit Docker config directories are used', async () => { + // Setup -- Mock that we have CDK credentials, and mock fs operations. + jest.spyOn(dockercreds, 'cdkCredentialsConfig').mockReturnValue({ version: '0.1', domainCredentials: {} }); + jest.spyOn(fs, 'mkdtempSync').mockImplementationOnce(() => '/tmp/mockedTempDir'); + jest.spyOn(fs, 'writeFileSync').mockImplementation(jest.fn()); + + let pub = new AssetPublishing(AssetManifest.fromPath('/simple/cdk.out'), { aws }); + aws.mockEcr.describeImages = mockedApiFailure('ImageNotFoundException', 'File does not exist'); + aws.mockEcr.getAuthorizationToken = mockedApiResult({ + authorizationData: [ + { authorizationToken: 'dXNlcjpwYXNz', proxyEndpoint: 'https://proxy.com/' }, + ], + }); + + const expectAllSpawns = mockSpawn( + // Initally use the first created directory with the CDK credentials + { commandLine: ['docker', '--config', '/tmp/mockedTempDir', 'inspect', 'cdkasset-theasset'], exitCode: 1 }, + { commandLine: ['docker', '--config', '/tmp/mockedTempDir', 'build', '--tag', 'cdkasset-theasset', '.'], cwd: absoluteDockerPath }, + { commandLine: ['docker', '--config', '/tmp/mockedTempDir', 'tag', 'cdkasset-theasset', '12345.amazonaws.com/repo:abcdef'] }, + // Prior to push, revert to the default config directory + { commandLine: ['docker', 'login'], prefix: true }, + { commandLine: ['docker', 'push', '12345.amazonaws.com/repo:abcdef'] }, + ); + + await pub.publish(); + + expectAllSpawns(); +}); + +test('logging in only once for two assets', async () => { + const pub = new AssetPublishing(AssetManifest.fromPath('/multi/cdk.out'), { aws, throwOnError: false }); + aws.mockEcr.describeImages = mockedApiFailure('ImageNotFoundException', 'File does not exist'); + aws.mockEcr.getAuthorizationToken = mockedApiResult({ + authorizationData: [ + { authorizationToken: 'dXNlcjpwYXNz', proxyEndpoint: 'https://proxy.com/' }, + ], + }); + + const expectAllSpawns = mockSpawn( + { commandLine: ['docker', 'login', '--username', 'user', '--password-stdin', 'https://proxy.com/'] }, + { commandLine: ['docker', 'inspect', 'cdkasset-theasset1'], exitCode: 1 }, + { commandLine: ['docker', 'build', '--tag', 'cdkasset-theasset1', '.'], cwd: '/multi/cdk.out/dockerdir' }, + { commandLine: ['docker', 'tag', 'cdkasset-theasset1', '12345.amazonaws.com/repo:theAsset1'] }, + { commandLine: ['docker', 'push', '12345.amazonaws.com/repo:theAsset1'] }, + { commandLine: ['docker', 'inspect', 'cdkasset-theasset2'], exitCode: 1 }, + { commandLine: ['docker', 'build', '--tag', 'cdkasset-theasset2', '.'], cwd: '/multi/cdk.out/dockerdir' }, + { commandLine: ['docker', 'tag', 'cdkasset-theasset2', '12345.amazonaws.com/repo:theAsset2'] }, + { commandLine: ['docker', 'push', '12345.amazonaws.com/repo:theAsset2'] }, + ); + + await pub.publish(); + + expectAllSpawns(); + expect(true).toBeTruthy(); // Expect no exception, satisfy linter +}); + +test('logging in twice for two repository domains (containing account id & region)', async () => { + const pub = new AssetPublishing(AssetManifest.fromPath('/multi/cdk.out'), { aws, throwOnError: false }); + aws.mockEcr.describeImages = mockedApiFailure('ImageNotFoundException', 'File does not exist'); + + let repoIdx = 12345; + aws.mockEcr.describeRepositories = jest.fn().mockReturnValue({ + promise: jest.fn().mockImplementation(() => Promise.resolve({ + repositories: [ + // Usually looks like: 012345678910.dkr.ecr.us-west-2.amazonaws.com/aws-cdk/assets + { repositoryUri: `${repoIdx++}.amazonaws.com/aws-cdk/assets` }, + ], + })), + }); + + let proxyIdx = 12345; + aws.mockEcr.getAuthorizationToken = jest.fn().mockReturnValue({ + promise: jest.fn().mockImplementation(() => Promise.resolve({ + authorizationData: [ + { authorizationToken: 'dXNlcjpwYXNz', proxyEndpoint: `https://${proxyIdx++}.proxy.com/` }, + ], + })), + }); + + const expectAllSpawns = mockSpawn( + { commandLine: ['docker', 'login', '--username', 'user', '--password-stdin', 'https://12345.proxy.com/'] }, + { commandLine: ['docker', 'inspect', 'cdkasset-theasset1'], exitCode: 1 }, + { commandLine: ['docker', 'build', '--tag', 'cdkasset-theasset1', '.'], cwd: '/multi/cdk.out/dockerdir' }, + { commandLine: ['docker', 'tag', 'cdkasset-theasset1', '12345.amazonaws.com/aws-cdk/assets:theAsset1'] }, + { commandLine: ['docker', 'push', '12345.amazonaws.com/aws-cdk/assets:theAsset1'] }, + { commandLine: ['docker', 'login', '--username', 'user', '--password-stdin', 'https://12346.proxy.com/'] }, + { commandLine: ['docker', 'inspect', 'cdkasset-theasset2'], exitCode: 1 }, + { commandLine: ['docker', 'build', '--tag', 'cdkasset-theasset2', '.'], cwd: '/multi/cdk.out/dockerdir' }, + { commandLine: ['docker', 'tag', 'cdkasset-theasset2', '12346.amazonaws.com/aws-cdk/assets:theAsset2'] }, + { commandLine: ['docker', 'push', '12346.amazonaws.com/aws-cdk/assets:theAsset2'] }, + ); + + await pub.publish(); + + expectAllSpawns(); + expect(true).toBeTruthy(); // Expect no exception, satisfy linter +}); + +test('building only', async () => { + const pub = new AssetPublishing(AssetManifest.fromPath('/multi/cdk.out'), { + aws, + throwOnError: false, + buildAssets: true, + publishAssets: false, + }); + + aws.mockEcr.describeImages = mockedApiFailure('ImageNotFoundException', 'File does not exist'); + aws.mockEcr.getAuthorizationToken = mockedApiResult({ + authorizationData: [ + { authorizationToken: 'dXNlcjpwYXNz', proxyEndpoint: 'https://proxy.com/' }, + ], + }); + + const expectAllSpawns = mockSpawn( + { commandLine: ['docker', 'login', '--username', 'user', '--password-stdin', 'https://proxy.com/'] }, + { commandLine: ['docker', 'inspect', 'cdkasset-theasset1'], exitCode: 1 }, + { commandLine: ['docker', 'build', '--tag', 'cdkasset-theasset1', '.'], cwd: '/multi/cdk.out/dockerdir' }, + { commandLine: ['docker', 'tag', 'cdkasset-theasset1', '12345.amazonaws.com/repo:theAsset1'] }, + { commandLine: ['docker', 'inspect', 'cdkasset-theasset2'], exitCode: 1 }, + { commandLine: ['docker', 'build', '--tag', 'cdkasset-theasset2', '.'], cwd: '/multi/cdk.out/dockerdir' }, + { commandLine: ['docker', 'tag', 'cdkasset-theasset2', '12345.amazonaws.com/repo:theAsset2'] }, + ); + + await pub.publish(); + + expectAllSpawns(); + expect(true).toBeTruthy(); // Expect no exception, satisfy linter +}); + +test('publishing only', async () => { + const pub = new AssetPublishing(AssetManifest.fromPath('/multi/cdk.out'), { + aws, + throwOnError: false, + buildAssets: false, + publishAssets: true, + }); + + aws.mockEcr.describeImages = mockedApiFailure('ImageNotFoundException', 'File does not exist'); + aws.mockEcr.getAuthorizationToken = mockedApiResult({ + authorizationData: [ + { authorizationToken: 'dXNlcjpwYXNz', proxyEndpoint: 'https://proxy.com/' }, + ], + }); + + const expectAllSpawns = mockSpawn( + { commandLine: ['docker', 'login', '--username', 'user', '--password-stdin', 'https://proxy.com/'] }, + { commandLine: ['docker', 'push', '12345.amazonaws.com/aws-cdk/assets:theAsset1'] }, + { commandLine: ['docker', 'push', '12345.amazonaws.com/aws-cdk/assets:theAsset2'] }, + ); + + await pub.publish(); + + expectAllSpawns(); + expect(true).toBeTruthy(); // Expect no exception, satisfy linter +}); + +test('overriding the docker command', async () => { + process.env.CDK_DOCKER = 'custom'; + + const pub = new AssetPublishing(AssetManifest.fromPath('/simple/cdk.out'), { aws, throwOnError: false }); + + aws.mockEcr.describeImages = mockedApiFailure('ImageNotFoundException', 'File does not exist'); + aws.mockEcr.getAuthorizationToken = mockedApiResult({ + authorizationData: [ + { authorizationToken: 'dXNlcjpwYXNz', proxyEndpoint: 'https://proxy.com/' }, + ], + }); + + const expectAllSpawns = mockSpawn( + { commandLine: ['custom', 'login', '--username', 'user', '--password-stdin', 'https://proxy.com/'] }, + { commandLine: ['custom', 'inspect', 'cdkasset-theasset'] }, + { commandLine: ['custom', 'tag', 'cdkasset-theasset', '12345.amazonaws.com/repo:abcdef'] }, + { commandLine: ['custom', 'push', '12345.amazonaws.com/repo:abcdef'] }, + ); + + await pub.publish(); + + expectAllSpawns(); + expect(true).toBeTruthy(); // Expect no exception, satisfy linter +}); diff --git a/packages/cdk-bogus/test/fake-listener.ts b/packages/cdk-bogus/test/fake-listener.ts new file mode 100644 index 0000000000000..7aef7fbe9d9f6 --- /dev/null +++ b/packages/cdk-bogus/test/fake-listener.ts @@ -0,0 +1,17 @@ +import { IPublishProgressListener, EventType, IPublishProgress } from '../lib/progress'; + +export class FakeListener implements IPublishProgressListener { + public readonly types = new Array(); + public readonly messages = new Array(); + + constructor(private readonly doAbort = false) { + } + + public onPublishEvent(_type: EventType, event: IPublishProgress): void { + this.messages.push(event.message); + + if (this.doAbort) { + event.abort(); + } + } +} \ No newline at end of file diff --git a/packages/cdk-bogus/test/files.test.ts b/packages/cdk-bogus/test/files.test.ts new file mode 100644 index 0000000000000..83af51717bea6 --- /dev/null +++ b/packages/cdk-bogus/test/files.test.ts @@ -0,0 +1,344 @@ +jest.mock('child_process'); + +import { Manifest } from '@aws-cdk/cloud-assembly-schema'; +import * as mockfs from 'mock-fs'; +import { FakeListener } from './fake-listener'; +import { mockAws, mockedApiFailure, mockedApiResult, mockUpload } from './mock-aws'; +import { mockSpawn } from './mock-child_process'; +import { AssetPublishing, AssetManifest } from '../lib'; + +const ABS_PATH = '/simple/cdk.out/some_external_file'; + +const DEFAULT_DESTINATION = { + region: 'us-north-50', + assumeRoleArn: 'arn:aws:role', + bucketName: 'some_bucket', + objectKey: 'some_key', +}; + +let aws: ReturnType; +beforeEach(() => { + jest.resetAllMocks(); + + mockfs({ + '/simple/cdk.out/assets.json': JSON.stringify({ + version: Manifest.version(), + files: { + theAsset: { + source: { + path: 'some_file', + }, + destinations: { theDestination: DEFAULT_DESTINATION }, + }, + }, + }), + '/simple/cdk.out/some_file': 'FILE_CONTENTS', + [ABS_PATH]: 'ZIP_FILE_THAT_IS_DEFINITELY_NOT_EMPTY', + '/abs/cdk.out/assets.json': JSON.stringify({ + version: Manifest.version(), + files: { + theAsset: { + source: { + path: '/simple/cdk.out/some_file', + }, + destinations: { theDestination: { ...DEFAULT_DESTINATION, bucketName: 'some_other_bucket' } }, + }, + }, + }), + '/external/cdk.out/assets.json': JSON.stringify({ + version: Manifest.version(), + files: { + externalAsset: { + source: { + executable: ['sometool'], + }, + destinations: { theDestination: { ...DEFAULT_DESTINATION, bucketName: 'some_external_bucket' } }, + }, + }, + }), + '/types/cdk.out/assets.json': JSON.stringify({ + version: Manifest.version(), + files: { + theTextAsset: { + source: { + path: 'plain_text.txt', + }, + destinations: { theDestination: { ...DEFAULT_DESTINATION, objectKey: 'some_key.txt' } }, + }, + theImageAsset: { + source: { + path: 'image.png', + }, + destinations: { theDestination: { ...DEFAULT_DESTINATION, objectKey: 'some_key.png' } }, + }, + }, + }), + '/types/cdk.out/plain_text.txt': 'FILE_CONTENTS', + '/types/cdk.out/image.png': 'FILE_CONTENTS', + '/emptyzip/cdk.out/assets.json': JSON.stringify({ + version: Manifest.version(), + files: { + theTextAsset: { + source: { + path: 'empty_dir', + packaging: 'zip', + }, + destinations: { theDestination: DEFAULT_DESTINATION }, + }, + }, + }), + '/emptyzip/cdk.out/empty_dir': { }, // Empty directory + }); + + aws = mockAws(); +}); + +afterEach(() => { + mockfs.restore(); +}); + +test('pass destination properties to AWS client', async () => { + const pub = new AssetPublishing(AssetManifest.fromPath('/simple/cdk.out'), { aws, throwOnError: false }); + aws.mockS3.listObjectsV2 = mockedApiResult({}); + + await pub.publish(); + + expect(aws.s3Client).toHaveBeenCalledWith(expect.objectContaining({ + region: 'us-north-50', + assumeRoleArn: 'arn:aws:role', + })); +}); + +test('Do nothing if file already exists', async () => { + const pub = new AssetPublishing(AssetManifest.fromPath('/simple/cdk.out'), { aws }); + + aws.mockS3.listObjectsV2 = mockedApiResult({ Contents: [{ Key: 'some_key' }] }); + aws.mockS3.upload = mockUpload(); + await pub.publish(); + + expect(aws.mockS3.listObjectsV2).toHaveBeenCalledWith(expect.objectContaining({ + Bucket: 'some_bucket', + Prefix: 'some_key', + MaxKeys: 1, + })); + expect(aws.mockS3.upload).not.toHaveBeenCalled(); +}); + +test('tiny file does not count as cache hit', async () => { + const pub = new AssetPublishing(AssetManifest.fromPath('/simple/cdk.out'), { aws }); + + aws.mockS3.listObjectsV2 = mockedApiResult({ Contents: [{ Key: 'some_key', Size: 5 }] }); + aws.mockS3.upload = mockUpload(); + + await pub.publish(); + + expect(aws.mockS3.upload).toHaveBeenCalled(); +}); + +test('upload file if new (list returns other key)', async () => { + const pub = new AssetPublishing(AssetManifest.fromPath('/simple/cdk.out'), { aws }); + + aws.mockS3.listObjectsV2 = mockedApiResult({ Contents: [{ Key: 'some_key.but_not_the_one' }] }); + aws.mockS3.upload = mockUpload('FILE_CONTENTS'); + + await pub.publish(); + + expect(aws.mockS3.upload).toHaveBeenCalledWith(expect.objectContaining({ + Bucket: 'some_bucket', + Key: 'some_key', + ContentType: 'application/octet-stream', + })); + + // We'll just have to assume the contents are correct +}); + +test('upload with server side encryption AES256 header', async () => { + const pub = new AssetPublishing(AssetManifest.fromPath('/simple/cdk.out'), { aws }); + + aws.mockS3.getBucketEncryption = mockedApiResult({ + ServerSideEncryptionConfiguration: { + Rules: [ + { + ApplyServerSideEncryptionByDefault: { + SSEAlgorithm: 'AES256', + }, + BucketKeyEnabled: false, + }, + ], + }, + }); + aws.mockS3.listObjectsV2 = mockedApiResult({ Contents: [{ Key: 'some_key.but_not_the_one' }] }); + aws.mockS3.upload = mockUpload('FILE_CONTENTS'); + + await pub.publish(); + + expect(aws.mockS3.upload).toHaveBeenCalledWith(expect.objectContaining({ + Bucket: 'some_bucket', + Key: 'some_key', + ContentType: 'application/octet-stream', + ServerSideEncryption: 'AES256', + })); + + // We'll just have to assume the contents are correct +}); + +test('upload with server side encryption aws:kms header and key id', async () => { + const pub = new AssetPublishing(AssetManifest.fromPath('/simple/cdk.out'), { aws }); + + aws.mockS3.getBucketEncryption = mockedApiResult({ + ServerSideEncryptionConfiguration: { + Rules: [ + { + ApplyServerSideEncryptionByDefault: { + SSEAlgorithm: 'aws:kms', + KMSMasterKeyID: 'the-key-id', + }, + BucketKeyEnabled: false, + }, + ], + }, + }); + + aws.mockS3.listObjectsV2 = mockedApiResult({ Contents: [{ Key: 'some_key.but_not_the_one' }] }); + aws.mockS3.upload = mockUpload('FILE_CONTENTS'); + + await pub.publish(); + + expect(aws.mockS3.upload).toHaveBeenCalledWith(expect.objectContaining({ + Bucket: 'some_bucket', + Key: 'some_key', + ContentType: 'application/octet-stream', + ServerSideEncryption: 'aws:kms', + SSEKMSKeyId: 'the-key-id', + })); + + // We'll just have to assume the contents are correct +}); + +test('will only read bucketEncryption once even for multiple assets', async () => { + const pub = new AssetPublishing(AssetManifest.fromPath('/types/cdk.out'), { aws }); + + aws.mockS3.listObjectsV2 = mockedApiResult({ Contents: [{ Key: 'some_key.but_not_the_one' }] }); + aws.mockS3.upload = mockUpload('FILE_CONTENTS'); + + await pub.publish(); + + expect(aws.mockS3.upload).toHaveBeenCalledTimes(2); + expect(aws.mockS3.getBucketEncryption).toHaveBeenCalledTimes(1); +}); + +test('no server side encryption header if access denied for bucket encryption', async () => { + const progressListener = new FakeListener(); + const pub = new AssetPublishing(AssetManifest.fromPath('/simple/cdk.out'), { aws, progressListener }); + + aws.mockS3.getBucketEncryption = mockedApiFailure('AccessDenied', 'Access Denied'); + + aws.mockS3.listObjectsV2 = mockedApiResult({ Contents: [{ Key: 'some_key.but_not_the_one' }] }); + aws.mockS3.upload = mockUpload('FILE_CONTENTS'); + + await pub.publish(); + + expect(aws.mockS3.upload).toHaveBeenCalledWith(expect.not.objectContaining({ + ServerSideEncryption: 'aws:kms', + })); + + expect(aws.mockS3.upload).toHaveBeenCalledWith(expect.not.objectContaining({ + ServerSideEncryption: 'AES256', + })); +}); + +test('correctly looks up content type', async () => { + const pub = new AssetPublishing(AssetManifest.fromPath('/types/cdk.out'), { aws }); + + aws.mockS3.listObjectsV2 = mockedApiResult({ Contents: [{ Key: 'some_key.but_not_the_one' }] }); + aws.mockS3.upload = mockUpload('FILE_CONTENTS'); + + await pub.publish(); + + expect(aws.mockS3.upload).toHaveBeenCalledWith(expect.objectContaining({ + Bucket: 'some_bucket', + Key: 'some_key.txt', + ContentType: 'text/plain', + })); + + expect(aws.mockS3.upload).toHaveBeenCalledWith(expect.objectContaining({ + Bucket: 'some_bucket', + Key: 'some_key.png', + ContentType: 'image/png', + })); + + // We'll just have to assume the contents are correct +}); + +test('upload file if new (list returns no key)', async () => { + const pub = new AssetPublishing(AssetManifest.fromPath('/simple/cdk.out'), { aws }); + + aws.mockS3.listObjectsV2 = mockedApiResult({ Contents: undefined }); + aws.mockS3.upload = mockUpload('FILE_CONTENTS'); + + await pub.publish(); + + expect(aws.mockS3.upload).toHaveBeenCalledWith(expect.objectContaining({ + Bucket: 'some_bucket', + Key: 'some_key', + })); + + // We'll just have to assume the contents are correct +}); + +test('successful run does not need to query account ID', async () => { + const pub = new AssetPublishing(AssetManifest.fromPath('/simple/cdk.out'), { aws }); + + aws.mockS3.listObjectsV2 = mockedApiResult({ Contents: undefined }); + aws.mockS3.upload = mockUpload('FILE_CONTENTS'); + + await pub.publish(); + + expect(aws.discoverCurrentAccount).not.toHaveBeenCalled(); + expect(aws.discoverTargetAccount).not.toHaveBeenCalled(); +}); + +test('correctly identify asset path if path is absolute', async () => { + const pub = new AssetPublishing(AssetManifest.fromPath('/abs/cdk.out'), { aws }); + + aws.mockS3.listObjectsV2 = mockedApiResult({ Contents: undefined }); + aws.mockS3.upload = mockUpload('FILE_CONTENTS'); + + await pub.publish(); + + expect(true).toBeTruthy(); // No exception, satisfy linter +}); + +describe('external assets', () => { + let pub: AssetPublishing; + beforeEach(() => { + pub = new AssetPublishing(AssetManifest.fromPath('/external/cdk.out'), { aws }); + }); + + test('do nothing if file exists already', async () => { + aws.mockS3.listObjectsV2 = mockedApiResult({ Contents: [{ Key: 'some_key' }] }); + + await pub.publish(); + + expect(aws.mockS3.listObjectsV2).toHaveBeenCalledWith(expect.objectContaining({ + Bucket: 'some_external_bucket', + Prefix: 'some_key', + MaxKeys: 1, + })); + }); + + test('upload external asset correctly', async () => { + aws.mockS3.listObjectsV2 = mockedApiResult({ Contents: undefined }); + aws.mockS3.upload = mockUpload('ZIP_FILE_THAT_IS_DEFINITELY_NOT_EMPTY'); + const expectAllSpawns = mockSpawn({ commandLine: ['sometool'], stdout: ABS_PATH }); + + await pub.publish(); + + expect(aws.s3Client).toHaveBeenCalledWith(expect.objectContaining({ + region: 'us-north-50', + assumeRoleArn: 'arn:aws:role', + })); + + expectAllSpawns(); + }); +}); diff --git a/packages/cdk-bogus/test/manifest.test.ts b/packages/cdk-bogus/test/manifest.test.ts new file mode 100644 index 0000000000000..605d6922b5e08 --- /dev/null +++ b/packages/cdk-bogus/test/manifest.test.ts @@ -0,0 +1,117 @@ +import { Manifest } from '@aws-cdk/cloud-assembly-schema'; +import * as mockfs from 'mock-fs'; +import { AssetManifest, DestinationIdentifier, DestinationPattern, DockerImageManifestEntry, FileManifestEntry } from '../lib'; + +beforeEach(() => { + mockfs({ + '/simple/cdk.out/assets.json': JSON.stringify({ + version: Manifest.version(), + files: { + asset1: { + type: 'file', + source: { path: 'S1' }, + destinations: { + dest1: { bucketName: 'D1', objectKey: 'X' }, + dest2: { bucketName: 'D2', objectKey: 'X' }, + }, + }, + }, + dockerImages: { + asset2: { + type: 'thing', + source: { directory: 'S2' }, + destinations: { + dest1: { repositoryName: 'D3', imageTag: 'X' }, + dest2: { repositoryName: 'D4', imageTag: 'X' }, + }, + }, + }, + }), + }); +}); + +afterEach(() => { + mockfs.restore(); +}); + +test('Can list manifest', () => { + const manifest = AssetManifest.fromPath('/simple/cdk.out'); + expect(manifest.list().join('\n')).toEqual(` +asset1 file {\"path\":\"S1\"} + ├ asset1:dest1 {\"bucketName\":\"D1\",\"objectKey\":\"X\"} + └ asset1:dest2 {\"bucketName\":\"D2\",\"objectKey\":\"X\"} +asset2 docker-image {\"directory\":\"S2\"} + ├ asset2:dest1 {\"repositoryName\":\"D3\",\"imageTag\":\"X\"} + └ asset2:dest2 {\"repositoryName\":\"D4\",\"imageTag\":\"X\"} +`.trim()); +}); + +test('.entries() iterates over all destinations', () => { + const manifest = AssetManifest.fromPath('/simple/cdk.out'); + + expect(manifest.entries).toEqual([ + new FileManifestEntry(new DestinationIdentifier('asset1', 'dest1'), { path: 'S1' }, { bucketName: 'D1', objectKey: 'X' }), + new FileManifestEntry(new DestinationIdentifier('asset1', 'dest2'), { path: 'S1' }, { bucketName: 'D2', objectKey: 'X' }), + new DockerImageManifestEntry(new DestinationIdentifier('asset2', 'dest1'), { directory: 'S2' }, { repositoryName: 'D3', imageTag: 'X' }), + new DockerImageManifestEntry(new DestinationIdentifier('asset2', 'dest2'), { directory: 'S2' }, { repositoryName: 'D4', imageTag: 'X' }), + ]); +}); + +test('can select by asset ID', () => { + const manifest = AssetManifest.fromPath('/simple/cdk.out'); + + const subset = manifest.select([DestinationPattern.parse('asset2')]); + + expect(subset.entries.map(e => f(e.genericDestination, 'repositoryName'))).toEqual(['D3', 'D4']); +}); + +test('can select by asset ID + destination ID', () => { + const manifest = AssetManifest.fromPath('/simple/cdk.out'); + + const subset = manifest.select([ + DestinationPattern.parse('asset1:dest1'), + DestinationPattern.parse('asset2:dest2'), + ]); + + expect(subset.entries.map(e => f(e.genericDestination, 'repositoryName', 'bucketName'))).toEqual(['D1', 'D4']); +}); + +test('can select by destination ID', () => { + const manifest = AssetManifest.fromPath('/simple/cdk.out'); + + const subset = manifest.select([ + DestinationPattern.parse(':dest1'), + ]); + + expect(subset.entries.map(e => f(e.genericDestination, 'repositoryName', 'bucketName'))).toEqual(['D1', 'D3']); +}); + +test('empty string is not a valid pattern', () => { + expect(() => { + DestinationPattern.parse(''); + }).toThrow(/Empty string is not a valid destination identifier/); +}); + +test('pattern must have two components', () => { + expect(() => { + DestinationPattern.parse('a:b:c'); + }).toThrow(/Asset identifier must contain at most 2/); +}); + +test('parse ASSET:* the same as ASSET and ASSET:', () => { + expect(DestinationPattern.parse('a:*')).toEqual(DestinationPattern.parse('a')); + expect(DestinationPattern.parse('a:*')).toEqual(DestinationPattern.parse('a:')); +}); + +test('parse *:DEST the same as :DEST', () => { + expect(DestinationPattern.parse('*:a')).toEqual(DestinationPattern.parse(':a')); +}); + +function f(obj: unknown, ...keys: string[]): any { + for (const k of keys) { + if (typeof obj === 'object' && obj !== null && k in obj) { + return (obj as any)[k]; + } + } + return undefined; +} diff --git a/packages/cdk-bogus/test/mock-aws.ts b/packages/cdk-bogus/test/mock-aws.ts new file mode 100644 index 0000000000000..10cb26da99727 --- /dev/null +++ b/packages/cdk-bogus/test/mock-aws.ts @@ -0,0 +1,74 @@ +jest.mock('aws-sdk'); +import * as AWS from 'aws-sdk'; + +export function mockAws() { + const mockEcr = new AWS.ECR(); + const mockS3 = new AWS.S3(); + const mockSecretsManager = new AWS.SecretsManager(); + + // Sane defaults which can be overridden + mockS3.getBucketLocation = mockedApiResult({}); + mockS3.getBucketEncryption = mockedApiResult({}); + mockEcr.describeRepositories = mockedApiResult({ + repositories: [ + { + repositoryUri: '12345.amazonaws.com/repo', + }, + ], + }); + mockSecretsManager.getSecretValue = mockedApiFailure('NotImplemented', 'You need to supply an implementation for getSecretValue'); + + return { + mockEcr, + mockS3, + mockSecretsManager, + discoverPartition: jest.fn(() => Promise.resolve('swa')), + discoverCurrentAccount: jest.fn(() => Promise.resolve({ accountId: 'current_account', partition: 'swa' })), + discoverDefaultRegion: jest.fn(() => Promise.resolve('current_region')), + discoverTargetAccount: jest.fn(() => Promise.resolve({ accountId: 'target_account', partition: 'swa' })), + ecrClient: jest.fn(() => Promise.resolve(mockEcr)), + s3Client: jest.fn(() => Promise.resolve(mockS3)), + secretsManagerClient: jest.fn(() => Promise.resolve(mockSecretsManager)), + }; +} + +export function errorWithCode(code: string, message: string) { + const ret = new Error(message); + (ret as any).code = code; + return ret; +} + +export function mockedApiResult(returnValue: any) { + return jest.fn().mockReturnValue({ + promise: jest.fn().mockResolvedValue(returnValue), + }); +} + +export function mockedApiFailure(code: string, message: string) { + return jest.fn().mockReturnValue({ + promise: jest.fn().mockRejectedValue(errorWithCode(code, message)), + }); +} + +/** + * Mock upload, draining the stream that we get before returning + * so no race conditions happen with the uninstallation of mock-fs. + */ +export function mockUpload(expectContent?: string) { + return jest.fn().mockImplementation(request => ({ + promise: () => new Promise((ok, ko) => { + const didRead = new Array(); + + const bodyStream: NodeJS.ReadableStream = request.Body; + bodyStream.on('data', (chunk) => { didRead.push(chunk.toString()); }); // This listener must exist + bodyStream.on('error', ko); + bodyStream.on('close', () => { + const actualContent = didRead.join(''); + if (expectContent !== undefined && expectContent !== actualContent) { + throw new Error(`Expected to read '${expectContent}' but read: '${actualContent}'`); + } + ok(); + }); + }), + })); +} diff --git a/packages/cdk-bogus/test/mock-child_process.ts b/packages/cdk-bogus/test/mock-child_process.ts new file mode 100644 index 0000000000000..2cb513e24fff7 --- /dev/null +++ b/packages/cdk-bogus/test/mock-child_process.ts @@ -0,0 +1,69 @@ +import * as child_process from 'child_process'; +import * as events from 'events'; + +if (!(child_process as any).spawn.mockImplementationOnce) { + throw new Error('Call "jest.mock(\'child_process\');" at the top of the test file!'); +} + +export interface Invocation { + commandLine: string[]; + cwd?: string; + exitCode?: number; + stdout?: string; + + /** + * Only match a prefix of the command (don't care about the details of the arguments) + */ + prefix?: boolean; +} + +export function mockSpawn(...invocations: Invocation[]): () => void { + let mock = (child_process.spawn as any); + for (const _invocation of invocations) { + const invocation = _invocation; // Mirror into variable for closure + mock = mock.mockImplementationOnce((binary: string, args: string[], options: child_process.SpawnOptions) => { + if (invocation.prefix) { + // Match command line prefix + expect([binary, ...args].slice(0, invocation.commandLine.length)).toEqual(invocation.commandLine); + } else { + // Match full command line + expect([binary, ...args]).toEqual(invocation.commandLine); + } + + if (invocation.cwd != null) { + expect(options.cwd).toBe(invocation.cwd); + } + + const child: any = new events.EventEmitter(); + child.stdin = new events.EventEmitter(); + child.stdin.write = jest.fn(); + child.stdin.end = jest.fn(); + child.stdout = new events.EventEmitter(); + child.stderr = new events.EventEmitter(); + + if (invocation.stdout) { + mockEmit(child.stdout, 'data', Buffer.from(invocation.stdout)); + } + mockEmit(child, 'close', invocation.exitCode ?? 0); + + return child; + }); + } + + mock.mockImplementation((binary: string, args: string[], _options: any) => { + throw new Error(`Did not expect call of ${JSON.stringify([binary, ...args])}`); + }); + + return () => { + expect(mock).toHaveBeenCalledTimes(invocations.length); + }; +} + +/** + * Must do this on the next tick, as emitter.emit() expects all listeners to have been attached already + */ +function mockEmit(emitter: events.EventEmitter, event: string, data: any) { + setImmediate(() => { + emitter.emit(event, data); + }); +} diff --git a/packages/cdk-bogus/test/placeholders.test.ts b/packages/cdk-bogus/test/placeholders.test.ts new file mode 100644 index 0000000000000..87944ca673c32 --- /dev/null +++ b/packages/cdk-bogus/test/placeholders.test.ts @@ -0,0 +1,82 @@ +import { Manifest } from '@aws-cdk/cloud-assembly-schema'; +import * as mockfs from 'mock-fs'; +import { mockAws, mockedApiResult } from './mock-aws'; +import { AssetManifest, AssetPublishing } from '../lib'; + +let aws: ReturnType; +beforeEach(() => { + mockfs({ + '/simple/cdk.out/assets.json': JSON.stringify({ + version: Manifest.version(), + files: { + fileAsset: { + type: 'file', + source: { + path: 'some_file', + }, + destinations: { + theDestination: { + // Absence of region + assumeRoleArn: 'arn:aws:role-${AWS::AccountId}', + bucketName: 'some_bucket-${AWS::AccountId}-${AWS::Region}', + objectKey: 'some_key-${AWS::AccountId}-${AWS::Region}', + }, + }, + }, + }, + dockerImages: { + dockerAsset: { + type: 'docker-image', + source: { + directory: 'dockerdir', + }, + destinations: { + theDestination: { + // Explicit region + region: 'explicit_region', + assumeRoleArn: 'arn:aws:role-${AWS::AccountId}', + repositoryName: 'repo-${AWS::AccountId}-${AWS::Region}', + imageTag: 'abcdef', + }, + }, + }, + }, + }), + '/simple/cdk.out/some_file': 'FILE_CONTENTS', + }); + + aws = mockAws(); +}); + +afterEach(() => { + mockfs.restore(); +}); + +test('check that placeholders are replaced', async () => { + const pub = new AssetPublishing(AssetManifest.fromPath('/simple/cdk.out'), { aws }); + aws.mockS3.getBucketLocation = mockedApiResult({}); + aws.mockS3.listObjectsV2 = mockedApiResult({ Contents: [{ Key: 'some_key-current_account-current_region' }] }); + aws.mockEcr.describeImages = mockedApiResult({ /* No error == image exists */ }); + + await pub.publish(); + + expect(aws.s3Client).toHaveBeenCalledWith(expect.objectContaining({ + assumeRoleArn: 'arn:aws:role-current_account', + })); + + expect(aws.ecrClient).toHaveBeenCalledWith(expect.objectContaining({ + region: 'explicit_region', + assumeRoleArn: 'arn:aws:role-current_account', + })); + + expect(aws.mockS3.listObjectsV2).toHaveBeenCalledWith(expect.objectContaining({ + Bucket: 'some_bucket-current_account-current_region', + Prefix: 'some_key-current_account-current_region', + MaxKeys: 1, + })); + + expect(aws.mockEcr.describeImages).toHaveBeenCalledWith(expect.objectContaining({ + imageIds: [{ imageTag: 'abcdef' }], + repositoryName: 'repo-current_account-explicit_region', + })); +}); diff --git a/packages/cdk-bogus/test/private/docker-credentials.test.ts b/packages/cdk-bogus/test/private/docker-credentials.test.ts new file mode 100644 index 0000000000000..c4c3f45ce9edf --- /dev/null +++ b/packages/cdk-bogus/test/private/docker-credentials.test.ts @@ -0,0 +1,220 @@ +import * as os from 'os'; +import * as path from 'path'; +import * as mockfs from 'mock-fs'; +import { cdkCredentialsConfig, cdkCredentialsConfigFile, DockerCredentialsConfig, fetchDockerLoginCredentials } from '../../lib/private/docker-credentials'; +import { mockAws, mockedApiFailure, mockedApiResult } from '../mock-aws'; + +const _ENV = process.env; + +let aws: ReturnType; +beforeEach(() => { + jest.resetModules(); + jest.resetAllMocks(); + + aws = mockAws(); + + process.env = { ..._ENV }; +}); + +afterEach(() => { + mockfs.restore(); + process.env = _ENV; +}); + +describe('cdkCredentialsConfigFile', () => { + test('Can be overridden by CDK_DOCKER_CREDS_FILE', () => { + const credsFile = '/tmp/insertfilenamehere_cdk_config.json'; + process.env.CDK_DOCKER_CREDS_FILE = credsFile; + + expect(cdkCredentialsConfigFile()).toEqual(credsFile); + }); + + test('Uses homedir if no process env is set', () => { + expect(cdkCredentialsConfigFile()).toEqual(path.join(os.userInfo().homedir, '.cdk', 'cdk-docker-creds.json')); + }); +}); + +describe('cdkCredentialsConfig', () => { + const credsFile = '/tmp/foo/bar/does/not/exist/config.json'; + beforeEach(() => { process.env.CDK_DOCKER_CREDS_FILE = credsFile; }); + + test('returns undefined if no config exists', () => { + expect(cdkCredentialsConfig()).toBeUndefined(); + }); + + test('returns parsed config if it exists', () => { + mockfs({ + [credsFile]: JSON.stringify({ + version: '0.1', + domainCredentials: { + 'test1.example.com': { secretsManagerSecretId: 'mySecret' }, + 'test2.example.com': { ecrRepository: 'arn:aws:ecr:bar' }, + }, + }), + }); + + const config = cdkCredentialsConfig(); + expect(config).toBeDefined(); + expect(config?.version).toEqual('0.1'); + expect(config?.domainCredentials['test1.example.com']?.secretsManagerSecretId).toEqual('mySecret'); + expect(config?.domainCredentials['test2.example.com']?.ecrRepository).toEqual('arn:aws:ecr:bar'); + }); +}); + +describe('fetchDockerLoginCredentials', () => { + let config: DockerCredentialsConfig; + + beforeEach(() => { + config = { + version: '0.1', + domainCredentials: { + 'misconfigured.example.com': {}, + 'secret.example.com': { secretsManagerSecretId: 'mySecret' }, + 'secretwithrole.example.com': { + secretsManagerSecretId: 'mySecret', + assumeRoleArn: 'arn:aws:iam::0123456789012:role/my-role', + }, + 'secretwithcustomfields.example.com': { + secretsManagerSecretId: 'mySecret', + secretsUsernameField: 'name', + secretsPasswordField: 'apiKey', + assumeRoleArn: 'arn:aws:iam::0123456789012:role/my-role', + }, + 'ecr.example.com': { ecrRepository: true }, + 'ecrwithrole.example.com': { + ecrRepository: true, + assumeRoleArn: 'arn:aws:iam::0123456789012:role/my-role', + }, + }, + }; + }); + + test('throws on unknown domain', async () => { + await expect(fetchDockerLoginCredentials(aws, config, 'unknowndomain.example.com')).rejects.toThrow(/unknown domain/); + }); + + test('throws on misconfigured domain (no ECR or SM)', async () => { + await expect(fetchDockerLoginCredentials(aws, config, 'misconfigured.example.com')).rejects.toThrow(/unknown credential type/); + }); + + test('does not throw on correctly configured raw domain', async () => { + mockSecretWithSecretString({ username: 'secretUser', secret: 'secretPass' }); + + await expect(fetchDockerLoginCredentials(aws, config, 'https://secret.example.com/v1/')).resolves.toBeTruthy(); + }); + + describe('SecretsManager', () => { + test('returns the credentials sucessfully if configured correctly - domain', async () => { + mockSecretWithSecretString({ username: 'secretUser', secret: 'secretPass' }); + + const creds = await fetchDockerLoginCredentials(aws, config, 'secret.example.com'); + + expect(creds).toEqual({ Username: 'secretUser', Secret: 'secretPass' }); + }); + + test('returns the credentials successfully if configured correctly - raw domain', async () => { + mockSecretWithSecretString({ username: 'secretUser', secret: 'secretPass' }); + + const creds = await fetchDockerLoginCredentials(aws, config, 'https://secret.example.com'); + + expect(creds).toEqual({ Username: 'secretUser', Secret: 'secretPass' }); + }); + + test('throws when SecretsManager returns an error', async () => { + const errMessage = "Secrets Manager can't find the specified secret."; + aws.mockSecretsManager.getSecretValue = mockedApiFailure('ResourceNotFoundException', errMessage); + + await expect(fetchDockerLoginCredentials(aws, config, 'secret.example.com')).rejects.toThrow(errMessage); + }); + + test('supports assuming a role', async () => { + mockSecretWithSecretString({ username: 'secretUser', secret: 'secretPass' }); + + const creds = await fetchDockerLoginCredentials(aws, config, 'secretwithrole.example.com'); + + expect(creds).toEqual({ Username: 'secretUser', Secret: 'secretPass' }); + expect(aws.secretsManagerClient).toHaveBeenCalledWith({ assumeRoleArn: 'arn:aws:iam::0123456789012:role/my-role' }); + }); + + test('supports configuring the secret fields', async () => { + mockSecretWithSecretString({ name: 'secretUser', apiKey: '01234567' }); + + const creds = await fetchDockerLoginCredentials(aws, config, 'secretwithcustomfields.example.com'); + + expect(creds).toEqual({ Username: 'secretUser', Secret: '01234567' }); + }); + + test('throws when secret does not have the correct fields - key/value', async () => { + mockSecretWithSecretString({ principal: 'foo', credential: 'bar' }); + + await expect(fetchDockerLoginCredentials(aws, config, 'secret.example.com')).rejects.toThrow(/malformed secret string/); + }); + + test('throws when secret does not have the correct fields - plaintext', async () => { + mockSecretWithSecretString('myAPIKey'); + + await expect(fetchDockerLoginCredentials(aws, config, 'secret.example.com')).rejects.toThrow(/malformed secret string/); + }); + }); + + describe('ECR getAuthorizationToken', () => { + test('returns the credentials successfully', async () => { + mockEcrAuthorizationData(Buffer.from('myFoo:myBar', 'utf-8').toString('base64')); + + const creds = await fetchDockerLoginCredentials(aws, config, 'ecr.example.com'); + + expect(creds).toEqual({ Username: 'myFoo', Secret: 'myBar' }); + }); + + test('throws if ECR errors', async () => { + aws.mockEcr.getAuthorizationToken = mockedApiFailure('ServerException', 'uhoh'); + + await expect(fetchDockerLoginCredentials(aws, config, 'ecr.example.com')).rejects.toThrow(/uhoh/); + }); + + test('supports assuming a role', async () => { + mockEcrAuthorizationData(Buffer.from('myFoo:myBar', 'utf-8').toString('base64')); + + const creds = await fetchDockerLoginCredentials(aws, config, 'ecrwithrole.example.com'); + + expect(creds).toEqual({ Username: 'myFoo', Secret: 'myBar' }); + expect(aws.ecrClient).toHaveBeenCalledWith({ assumeRoleArn: 'arn:aws:iam::0123456789012:role/my-role' }); + }); + + test('throws if ECR returns no authData', async () => { + aws.mockEcr.getAuthorizationToken = mockedApiResult({ authorizationData: [] }); + + await expect(fetchDockerLoginCredentials(aws, config, 'ecr.example.com')).rejects.toThrow(/No authorization data received from ECR/); + }); + + test('throws if ECR authData is in an incorrect format', async () => { + mockEcrAuthorizationData('notabase64encodedstring'); + + await expect(fetchDockerLoginCredentials(aws, config, 'ecr.example.com')).rejects.toThrow(/unexpected ECR authData format/); + }); + }); + +}); + +function mockSecretWithSecretString(secretString: any) { + aws.mockSecretsManager.getSecretValue = mockedApiResult({ + ARN: 'arn:aws:secretsmanager:eu-west-1:0123456789012:secret:mySecret', + Name: 'mySecret', + VersionId: 'fa81fe61-c167-4aca-969e-4d8df74d4814', + SecretString: JSON.stringify(secretString), + VersionStages: [ + 'AWSCURRENT', + ], + }); +} + +function mockEcrAuthorizationData(authorizationToken: string) { + aws.mockEcr.getAuthorizationToken = mockedApiResult({ + authorizationData: [ + { + authorizationToken, + proxyEndpoint: 'https://0123456789012.dkr.ecr.eu-west-1.amazonaws.com', + }, + ], + }); +} diff --git a/packages/cdk-bogus/test/private/docker.test.ts b/packages/cdk-bogus/test/private/docker.test.ts new file mode 100644 index 0000000000000..40c37ca35f271 --- /dev/null +++ b/packages/cdk-bogus/test/private/docker.test.ts @@ -0,0 +1,94 @@ +import { Docker } from '../../lib/private/docker'; +import { ShellOptions, ProcessFailedError } from '../../lib/private/shell'; + +type ShellExecuteMock = jest.SpyInstance, Parameters>; + +describe('Docker', () => { + describe('exists', () => { + let docker: Docker; + + const makeShellExecuteMock = ( + fn: (params: string[]) => void, + ): ShellExecuteMock => + jest.spyOn<{ execute: Docker['execute'] }, 'execute'>(Docker.prototype as any, 'execute').mockImplementation( + async (params: string[], _options?: ShellOptions) => fn(params), + ); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + beforeEach(() => { + docker = new Docker(); + }); + + test('returns true when image inspect command does not throw', async () => { + const spy = makeShellExecuteMock(() => undefined); + + const imageExists = await docker.exists('foo'); + + expect(imageExists).toBe(true); + expect(spy.mock.calls[0][0]).toEqual(['inspect', 'foo']); + }); + + test('throws when an arbitrary error is caught', async () => { + makeShellExecuteMock(() => { + throw new Error(); + }); + + await expect(docker.exists('foo')).rejects.toThrow(); + }); + + test('throws when the error is a shell failure but the exit code is unrecognized', async () => { + makeShellExecuteMock(() => { + throw new (class extends Error implements ProcessFailedError { + public readonly code = 'PROCESS_FAILED' + public readonly exitCode = 47 + public readonly signal = null + + constructor() { + super('foo'); + } + }); + }); + + await expect(docker.exists('foo')).rejects.toThrow(); + }); + + test('returns false when the error is a shell failure and the exit code is 1 (Docker)', async () => { + makeShellExecuteMock(() => { + throw new (class extends Error implements ProcessFailedError { + public readonly code = 'PROCESS_FAILED' + public readonly exitCode = 1 + public readonly signal = null + + constructor() { + super('foo'); + } + }); + }); + + const imageExists = await docker.exists('foo'); + + expect(imageExists).toBe(false); + }); + + test('returns false when the error is a shell failure and the exit code is 125 (Podman)', async () => { + makeShellExecuteMock(() => { + throw new (class extends Error implements ProcessFailedError { + public readonly code = 'PROCESS_FAILED' + public readonly exitCode = 125 + public readonly signal = null + + constructor() { + super('foo'); + } + }); + }); + + const imageExists = await docker.exists('foo'); + + expect(imageExists).toBe(false); + }); + }); +}); diff --git a/packages/cdk-bogus/test/progress.test.ts b/packages/cdk-bogus/test/progress.test.ts new file mode 100644 index 0000000000000..2cca80311942a --- /dev/null +++ b/packages/cdk-bogus/test/progress.test.ts @@ -0,0 +1,85 @@ +import { Manifest } from '@aws-cdk/cloud-assembly-schema'; +import * as mockfs from 'mock-fs'; +import { FakeListener } from './fake-listener'; +import { mockAws, mockedApiResult, mockUpload } from './mock-aws'; +import { AssetManifest, AssetPublishing } from '../lib'; + +let aws: ReturnType; +beforeEach(() => { + mockfs({ + '/simple/cdk.out/assets.json': JSON.stringify({ + version: Manifest.version(), + files: { + theAsset: { + source: { + path: 'some_file', + }, + destinations: { + theDestination1: { + region: 'us-north-50', + assumeRoleArn: 'arn:aws:role', + bucketName: 'some_bucket', + objectKey: 'some_key', + }, + theDestination2: { + region: 'us-north-50', + assumeRoleArn: 'arn:aws:role', + bucketName: 'some_bucket', + objectKey: 'some_key2', + }, + }, + }, + }, + }), + '/simple/cdk.out/some_file': 'FILE_CONTENTS', + }); + + aws = mockAws(); + + // Accept all S3 uploads as new + aws.mockS3.getBucketLocation = mockedApiResult({}); + aws.mockS3.listObjectsV2 = mockedApiResult({ Contents: undefined }); + aws.mockS3.upload = mockUpload(); +}); + +afterEach(() => { + mockfs.restore(); +}); + +test('test listener', async () => { + const progressListener = new FakeListener(); + + const pub = new AssetPublishing(AssetManifest.fromPath('/simple/cdk.out'), { aws, progressListener }); + await pub.publish(); + + const allMessages = progressListener.messages.join('\n'); + + // Log mentions asset/destination ids + expect(allMessages).toContain('theAsset:theDestination1'); + expect(allMessages).toContain('theAsset:theDestination2'); +}); + +test('test publishing in parallel', async () => { + const progressListener = new FakeListener(); + + const pub = new AssetPublishing(AssetManifest.fromPath('/simple/cdk.out'), { aws, progressListener, publishInParallel: true }); + await pub.publish(); + + const allMessages = progressListener.messages.join('\n'); + + // Log mentions asset/destination ids + expect(allMessages).toContain('theAsset:theDestination1'); + expect(allMessages).toContain('theAsset:theDestination2'); +}); + +test('test abort', async () => { + const progressListener = new FakeListener(true); + + const pub = new AssetPublishing(AssetManifest.fromPath('/simple/cdk.out'), { aws, progressListener }); + await pub.publish(); + + const allMessages = progressListener.messages.join('\n'); + + // We never get to asset 2 + expect(allMessages).not.toContain('theAsset:theDestination2'); +}); \ No newline at end of file diff --git a/packages/cdk-bogus/test/test-archive-follow/data/one.txt b/packages/cdk-bogus/test/test-archive-follow/data/one.txt new file mode 100644 index 0000000000000..56a6051ca2b02 --- /dev/null +++ b/packages/cdk-bogus/test/test-archive-follow/data/one.txt @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/packages/cdk-bogus/test/test-archive-follow/linked/two.txt b/packages/cdk-bogus/test/test-archive-follow/linked/two.txt new file mode 100644 index 0000000000000..d8263ee986059 --- /dev/null +++ b/packages/cdk-bogus/test/test-archive-follow/linked/two.txt @@ -0,0 +1 @@ +2 \ No newline at end of file diff --git a/packages/cdk-bogus/test/test-archive/executable.txt b/packages/cdk-bogus/test/test-archive/executable.txt new file mode 100755 index 0000000000000..e69de29bb2d1d diff --git a/packages/cdk-bogus/test/test-archive/file1.txt b/packages/cdk-bogus/test/test-archive/file1.txt new file mode 100644 index 0000000000000..7bb7edbd4b634 --- /dev/null +++ b/packages/cdk-bogus/test/test-archive/file1.txt @@ -0,0 +1 @@ +I am file1 \ No newline at end of file diff --git a/packages/cdk-bogus/test/test-archive/file2.txt b/packages/cdk-bogus/test/test-archive/file2.txt new file mode 100644 index 0000000000000..ccb69856e7157 --- /dev/null +++ b/packages/cdk-bogus/test/test-archive/file2.txt @@ -0,0 +1,2 @@ +I am file2 +BLA! \ No newline at end of file diff --git a/packages/cdk-bogus/test/test-archive/subdir/file3.txt b/packages/cdk-bogus/test/test-archive/subdir/file3.txt new file mode 100644 index 0000000000000..976606ef5a8ac --- /dev/null +++ b/packages/cdk-bogus/test/test-archive/subdir/file3.txt @@ -0,0 +1 @@ +I am in a subdirectory diff --git a/packages/cdk-bogus/test/util.test.ts b/packages/cdk-bogus/test/util.test.ts new file mode 100644 index 0000000000000..8e498076913f2 --- /dev/null +++ b/packages/cdk-bogus/test/util.test.ts @@ -0,0 +1,32 @@ +import { createCriticalSection } from '../lib/private/util'; + +test('critical section', async () => { + // GIVEN + const criticalSection = createCriticalSection(); + + // WHEN + const arr = new Array(); + void criticalSection(async () => { + await new Promise(res => setTimeout(res, 500)); + arr.push('first'); + }); + await criticalSection(async () => { + arr.push('second'); + }); + + // THEN + expect(arr).toEqual([ + 'first', + 'second', + ]); +}); + +test('exceptions in critical sections', async () => { + // GIVEN + const criticalSection = createCriticalSection(); + + // WHEN/THEN + await expect(() => criticalSection(async () => { + throw new Error('Thrown'); + })).rejects.toThrow('Thrown'); +}); \ No newline at end of file diff --git a/packages/cdk-bogus/test/zipping.test.ts b/packages/cdk-bogus/test/zipping.test.ts new file mode 100644 index 0000000000000..8b850896c1144 --- /dev/null +++ b/packages/cdk-bogus/test/zipping.test.ts @@ -0,0 +1,53 @@ +// Separate test file since the archiving module doesn't work well with 'mock-fs' +import { Manifest } from '@aws-cdk/cloud-assembly-schema'; +import * as bockfs from './bockfs'; +import { mockAws, mockedApiResult, mockUpload } from './mock-aws'; +import { AssetManifest, AssetPublishing } from '../lib'; + +let aws: ReturnType; +beforeEach(() => { + bockfs({ + '/simple/cdk.out/assets.json': JSON.stringify({ + version: Manifest.version(), + files: { + theAsset: { + source: { + path: 'some_dir', + packaging: 'zip', + }, + destinations: { + theDestination: { + region: 'us-north-50', + assumeRoleArn: 'arn:aws:role', + bucketName: 'some_bucket', + objectKey: 'some_key', + }, + }, + }, + }, + }), + '/simple/cdk.out/some_dir/some_file': 'FILE_CONTENTS', + }); + + aws = mockAws(); + + // Accept all S3 uploads as new + aws.mockS3.listObjectsV2 = mockedApiResult({ Contents: undefined }); + aws.mockS3.upload = mockUpload(); +}); + +afterEach(() => { + bockfs.restore(); +}); + +test('Take a zipped upload', async () => { + const pub = new AssetPublishing(AssetManifest.fromPath(bockfs.path('/simple/cdk.out')), { aws }); + + await pub.publish(); + + expect(aws.mockS3.upload).toHaveBeenCalledWith(expect.objectContaining({ + Bucket: 'some_bucket', + Key: 'some_key', + ContentType: 'application/zip', + })); +}); diff --git a/packages/cdk-bogus/tsconfig.json b/packages/cdk-bogus/tsconfig.json new file mode 100644 index 0000000000000..b238d46998d26 --- /dev/null +++ b/packages/cdk-bogus/tsconfig.json @@ -0,0 +1,28 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "commonjs", + "lib": ["es2020", "dom"], + "strict": true, + "alwaysStrict": true, + "declaration": true, + "inlineSourceMap": true, + "inlineSources": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "resolveJsonModule": true, + "composite": true, + "incremental": true + }, + "include": [ + "**/*.ts", + "**/*.d.ts", + "lib/init-templates/*/*/add-project.hook.ts" + ], + "exclude": [ + "lib/init-templates/*/typescript/**/*.ts" + ] +} +