feat(migrate): Add CDK Migrate --from-scan functionality

packages/@aws-cdk-testing/cli-integ/lib/with-cdk-app.ts

@@ -12,6 +12,7 @@ import { AwsContext, withAws } from './with-aws';
 import { withTimeout } from './with-timeout';
 
 export const DEFAULT_TEST_TIMEOUT_S = 10 * 60;
+export const EXTENDED_TEST_TIMEOUT_S = 30 * 60;
 
 /**
  * Higher order function to execute a block with a CDK app fixture
@@ -185,6 +186,10 @@ export function withDefaultFixture(block: (context: TestFixture) => Promise<void
   return withAws(withTimeout(DEFAULT_TEST_TIMEOUT_S, withCdkApp(block)));
 }
 
+export function withExtendedTimeoutFixture(block: (context: TestFixture) => Promise<void>) {
+  return withAws(withTimeout(EXTENDED_TEST_TIMEOUT_S, withCdkApp(block)));
+}
+
 export function withCDKMigrateFixture(language: string, block: (content: TestFixture) => Promise<void>) {
   return withAws(withTimeout(DEFAULT_TEST_TIMEOUT_S, withCdkMigrateApp(language, block)));
 }

packages/@aws-cdk-testing/cli-integ/resources/cdk-apps/app/app.js

@@ -86,7 +86,35 @@ class MigrateStack extends cdk.Stack {
         value: queue.node.defaultChild.logicalId,
       });
     }
+    if (process.env.SAMPLE_RESOURCES) {
+      const myTopic = new sns.Topic(this, 'migratetopic1', {
+        removalPolicy: cdk.RemovalPolicy.DESTROY,
+      });
+      cdk.Tags.of(myTopic).add('tag1', 'value1');
+      const myTopic2 = new sns.Topic(this, 'migratetopic2', {
+        removalPolicy: cdk.RemovalPolicy.DESTROY,
+      });
+      cdk.Tags.of(myTopic2).add('tag2', 'value2');
+      const myQueue = new sqs.Queue(this, 'migratequeue1', {
+        removalPolicy: cdk.RemovalPolicy.DESTROY,
+      });
+      cdk.Tags.of(myQueue).add('tag3', 'value3');
+    }
+    if (process.env.LAMBDA_RESOURCES) {
+      const myFunction = new lambda.Function(this, 'migratefunction1', {
+        code: lambda.Code.fromInline('console.log("hello world")'),
+        handler: 'index.handler',
+        runtime: lambda.Runtime.NODEJS_18_X,
+      });
+      cdk.Tags.of(myFunction).add('lambda-tag', 'lambda-value');
 
+      const myFunction2 = new lambda.Function(this, 'migratefunction2', {
+        code: lambda.Code.fromInline('console.log("hello world2")'),
+        handler: 'index.handler',
+        runtime: lambda.Runtime.NODEJS_18_X,
+      });
+      cdk.Tags.of(myFunction2).add('lambda-tag', 'lambda-value');
+    }
   }
 }
 

packages/@aws-cdk-testing/cli-integ/tests/cli-integ-tests/cli.integtest.ts

@@ -1,7 +1,7 @@
 import { promises as fs, existsSync } from 'fs';
 import * as os from 'os';
 import * as path from 'path';
-import { integTest, cloneDirectory, shell, withDefaultFixture, retry, sleep, randomInteger, withSamIntegrationFixture, RESOURCES_DIR, withCDKMigrateFixture } from '../../lib';
+import { integTest, cloneDirectory, shell, withDefaultFixture, retry, sleep, randomInteger, withSamIntegrationFixture, RESOURCES_DIR, withCDKMigrateFixture, withExtendedTimeoutFixture } from '../../lib';
 
 jest.setTimeout(2 * 60 * 60_000); // Includes the time to acquire locks, worst-case single-threaded runtime
 
@@ -571,9 +571,9 @@ integTest('deploy with role', withDefaultFixture(async (fixture) => {
   }
 }));
 
-// TODO add go back in when template synths properly
+// TODO add more testing that ensures the symmetry of the generated constructs to the resources.
 ['typescript', 'python', 'csharp', 'java'].forEach(language => {
-  integTest(`cdk migrate ${language}`, withCDKMigrateFixture(language, async (fixture) => {
+  integTest(`cdk migrate ${language} deploys successfully`, withCDKMigrateFixture(language, async (fixture) => {
     if (language === 'python') {
       await fixture.shell(['pip', 'install', '-r', 'requirements.txt']);
     }
@@ -588,6 +588,118 @@ integTest('deploy with role', withDefaultFixture(async (fixture) => {
   }));
 });
 
+integTest('cdk migrate generates migrate.json', withCDKMigrateFixture('typescript', async (fixture) => {
+
+  const migrateFile = await fs.readFile(path.join(fixture.integTestDir, 'migrate.json'), 'utf8');
+  const expectedFile = `{
+    \"//\": \"This file is generated by cdk migrate. It will be automatically deleted after the first successful deployment of this app to the environment of the original resources.\",
+    \"Source\": \"localfile\"
+  }`;
+  expect(JSON.parse(migrateFile)).toEqual(JSON.parse(expectedFile));
+  await fixture.cdkDestroy(fixture.stackNamePrefix);
+}));
+
+integTest('cdk migrate --from-scan with AND/OR filters correctly filters resources', withExtendedTimeoutFixture(async (fixture) => {
+  const stackName = `cdk-migrate-integ-${fixture.randomString}`;
+
+  await fixture.cdkDeploy('migrate-stack', {
+    modEnv: { SAMPLE_RESOURCES: '1' },
+  });
+  await fixture.cdk(
+    ['migrate', '--stack-name', stackName, '--from-scan', 'new', '--filter', 'type=AWS::SNS::Topic,tag-key=tag1', 'type=AWS::SQS::Queue,tag-key=tag3'],
+    { modEnv: { MIGRATE_INTEG_TEST: '1' }, neverRequireApproval: true, verbose: true, captureStderr: false },
+  );
+
+  try {
+    const response = await fixture.aws.cloudFormation('describeGeneratedTemplate', {
+      GeneratedTemplateName: stackName,
+    });
+    const resourceNames = [];
+    for (const resource of response.Resources || []) {
+      if (resource.LogicalResourceId) {
+        resourceNames.push(resource.LogicalResourceId);
+      }
+    }
+    fixture.log(`Resources: ${resourceNames}`);
+    expect(resourceNames.some(ele => ele && ele.includes('migratetopic1'))).toBeTruthy();
+    expect(resourceNames.some(ele => ele && ele.includes('migratequeue1'))).toBeTruthy();
+  } finally {
+    await fixture.cdkDestroy('migrate-stack');
+    await fixture.aws.cloudFormation('deleteGeneratedTemplate', {
+      GeneratedTemplateName: stackName,
+    });
+  }
+}));
+
+integTest('cdk migrate --from-scan for resources with Write Only Properties generates warnings', withExtendedTimeoutFixture(async (fixture) => {
+  const stackName = `cdk-migrate-integ-${fixture.randomString}`;
+
+  await fixture.cdkDeploy('migrate-stack', {
+    modEnv: {
+      LAMBDA_RESOURCES: '1',
+    },
+  });
+  await fixture.cdk(
+    ['migrate', '--stack-name', stackName, '--from-scan', 'new', '--filter', 'type=AWS::Lambda::Function,tag-key=lambda-tag'],
+    { modEnv: { MIGRATE_INTEG_TEST: '1' }, neverRequireApproval: true, verbose: true, captureStderr: false },
+  );
+
+  try {
+
+    const response = await fixture.aws.cloudFormation('describeGeneratedTemplate', {
+      GeneratedTemplateName: stackName,
+    });
+    const resourceNames = [];
+    for (const resource of response.Resources || []) {
+      if (resource.LogicalResourceId && resource.ResourceType === 'AWS::Lambda::Function') {
+        resourceNames.push(resource.LogicalResourceId);
+      }
+    }
+    fixture.log(`Resources: ${resourceNames}`);
+    const readmePath = path.join(fixture.integTestDir, stackName, 'README.md');
+    const readme = await fs.readFile(readmePath, 'utf8');
+    expect(readme).toContain('## Warnings');
+    for (const resourceName of resourceNames) {
+      expect(readme).toContain(`### ${resourceName}`);
+    }
+  } finally {
+    await fixture.cdkDestroy('migrate-stack');
+    await fixture.aws.cloudFormation('deleteGeneratedTemplate', {
+      GeneratedTemplateName: stackName,
+    });
+  }
+}));
+
+['typescript', 'python', 'csharp', 'java'].forEach(language => {
+  integTest(`cdk migrate --from-stack creates deployable ${language} app`, withExtendedTimeoutFixture(async (fixture) => {
+    const migrateStackName = fixture.fullStackName('migrate-stack');
+    await fixture.aws.cloudFormation('createStack', {
+      StackName: migrateStackName,
+      TemplateBody: await fs.readFile(path.join(__dirname, '..', '..', 'resources', 'templates', 'sqs-template.json'), 'utf8'),
+    });
+    try {
+      let stackStatus = 'CREATE_IN_PROGRESS';
+      while (stackStatus === 'CREATE_IN_PROGRESS') {
+        stackStatus = await (await (fixture.aws.cloudFormation('describeStacks', { StackName: migrateStackName }))).Stacks?.[0].StackStatus!;
+        await sleep(1000);
+      }
+      await fixture.cdk(
+        ['migrate', '--stack-name', migrateStackName, '--from-stack'],
+        { modEnv: { MIGRATE_INTEG_TEST: '1' }, neverRequireApproval: true, verbose: true, captureStderr: false },
+      );
+      await fixture.shell(['cd', path.join(fixture.integTestDir, migrateStackName)]);
+      await fixture.cdk(['deploy', migrateStackName], { neverRequireApproval: true, verbose: true, captureStderr: false });
+      const response = await fixture.aws.cloudFormation('describeStacks', {
+        StackName: migrateStackName,
+      });
+
+      expect(response.Stacks?.[0].StackStatus).toEqual('UPDATE_COMPLETE');
+    } finally {
+      await fixture.cdkDestroy('migrate-stack');
+    }
+  }));
+});
+
 integTest('cdk diff', withDefaultFixture(async (fixture) => {
   const diff1 = await fixture.cdk(['diff', fixture.fullStackName('test-1')]);
   expect(diff1).toContain('AWS::SNS::Topic');

packages/aws-cdk/lib/cdk-toolkit.ts

@@ -17,7 +17,7 @@ import { findCloudWatchLogGroups } from './api/logs/find-cloudwatch-logs';
 import { CloudWatchLogEventMonitor } from './api/logs/logs-monitor';
 import { createDiffChangeSet, ResourcesToImport } from './api/util/cloudformation';
 import { StackActivityProgress } from './api/util/cloudformation/stack-activity-monitor';
-import { generateCdkApp, generateStack, readFromPath, readFromStack, setEnvironment, validateSourceOptions } from './commands/migrate';
+import { generateCdkApp, generateStack, readFromPath, readFromStack, setEnvironment, parseSourceOptions, generateTemplate, FromScan, TemplateSourceOptions, GenerateTemplateOutput, CfnTemplateGeneratorProvider, writeMigrateJsonFile, buildGenertedTemplateOutput, buildCfnClient, appendWarningsToReadme, isThereAWarning } from './commands/migrate';
 import { printSecurityDiff, printStackDiff, RequireApproval } from './diff';
 import { ResourceImporter, removeNonImportResources } from './import';
 import { data, debug, error, highlight, print, success, warning, withCorkedLogging } from './logging';
@@ -735,19 +735,80 @@ export class CdkToolkit {
   public async migrate(options: MigrateOptions): Promise<void> {
     warning('This is an experimental feature and development on it is still in progress. We make no guarantees about the outcome or stability of the functionality.');
     const language = options.language?.toLowerCase() ?? 'typescript';
+    const environment = setEnvironment(options.account, options.region);
+    let generateTemplateOutput: GenerateTemplateOutput | undefined;
+    let cfn: CfnTemplateGeneratorProvider | undefined;
+    let templateToDelete: string | undefined;
 
     try {
-      validateSourceOptions(options.fromPath, options.fromStack);
-      const template = readFromPath(options.fromPath) ||
-        await readFromStack(options.stackName, this.props.sdkProvider, setEnvironment(options.account, options.region));
-      const stack = generateStack(template!, options.stackName, language);
+      // if neither fromPath nor fromStack is provided, generate a template using cloudformation
+      const scanType = parseSourceOptions(options.fromPath, options.fromStack, options.stackName).source;
+      if (scanType == TemplateSourceOptions.SCAN) {
+        generateTemplateOutput = await generateTemplate({
+          stackName: options.stackName,
+          filters: options.filter,
+          fromScan: options.fromScan,
+          sdkProvider: this.props.sdkProvider,
+          environment: environment,
+        });
+        templateToDelete = generateTemplateOutput.templateId;
+      } else if (scanType == TemplateSourceOptions.PATH) {
+        const templateBody = readFromPath(options.fromPath!);
+
+        const parsedTemplate = deserializeStructure(templateBody);
+        const templateId = parsedTemplate.Metadata?.TemplateId?.toString();
+        if (templateId) {
+          // if we have a template id, we can call describe generated template to get the resource identifiers
+          // resource metadata, and template source to generate the template
+          cfn = new CfnTemplateGeneratorProvider(await buildCfnClient(this.props.sdkProvider, environment));
+          const generatedTemplateSummary = await cfn.describeGeneratedTemplate(templateId);
+          generateTemplateOutput = buildGenertedTemplateOutput(generatedTemplateSummary, templateBody, generatedTemplateSummary.GeneratedTemplateId!);
+        } else {
+          generateTemplateOutput = {
+            migrateJson: {
+              templateBody: templateBody,
+              source: 'localfile',
+            },
+          };
+        }
+      } else if (scanType == TemplateSourceOptions.STACK) {
+        const template = await readFromStack(options.stackName, this.props.sdkProvider, environment);
+        if (!template) {
+          throw new Error(`No template found for stack-name: ${options.stackName}`);
+        }
+        generateTemplateOutput = {
+          migrateJson: {
+            templateBody: template,
+            source: options.stackName,
+          },
+        };
+      } else {
+        // We shouldn't ever get here, but just in case.
+        throw new Error(`Invalid source option provided: ${scanType}`);
+      }
+      const stack = generateStack(generateTemplateOutput.migrateJson.templateBody, options.stackName, language);
       success(' ⏳  Generating CDK app for %s...', chalk.blue(options.stackName));
       await generateCdkApp(options.stackName, stack!, language, options.outputPath, options.compress);
+      if (generateTemplateOutput) {
+        writeMigrateJsonFile(options.outputPath, options.stackName, generateTemplateOutput.migrateJson);
+      }
+      if (isThereAWarning(generateTemplateOutput)) {
+        warning(' ⚠️  Some resources could not be migrated completely. Please review the README.md file for more information.');
+        appendWarningsToReadme(`${path.join(options.outputPath ?? process.cwd(), options.stackName)}/README.md`, generateTemplateOutput.resources!);
+      }
     } catch (e) {
-      error(' ❌  Migrate failed for `%s`: %s', chalk.blue(options.stackName), (e as Error).message);
+      error(' ❌  Migrate failed for `%s`: %s', options.stackName, (e as Error).message);
       throw e;
+    } finally {
+      if (templateToDelete) {
+        if (!cfn) {
+          cfn = new CfnTemplateGeneratorProvider(await buildCfnClient(this.props.sdkProvider, environment));
+        }
+        if (!process.env.MIGRATE_INTEG_TEST) {
+          await cfn.deleteGeneratedTemplate(templateToDelete);
+        }
+      }
     }
-
   }
 
   private async selectStacksForList(patterns: string[]) {
@@ -1353,6 +1414,20 @@ export interface MigrateOptions {
    */
   readonly region?: string;
 
+  /**
+   * Filtering criteria used to select the resources to be included in the generated CDK app.
+   *
+   * @default - Include all resources
+   */
+  readonly filter?: string[];
+
+  /**
+   * Whether to initiate a new account scan for generating the CDK app.
+   *
+   * @default false
+   */
+  readonly fromScan?: FromScan;
+
   /**
    * Whether to zip the generated cdk app folder.
    *

packages/aws-cdk/lib/cli.ts

@@ -22,7 +22,7 @@ import { CdkToolkit, AssetBuildTime } from '../lib/cdk-toolkit';
 import { realHandler as context } from '../lib/commands/context';
 import { realHandler as docs } from '../lib/commands/docs';
 import { realHandler as doctor } from '../lib/commands/doctor';
-import { MIGRATE_SUPPORTED_LANGUAGES } from '../lib/commands/migrate';
+import { MIGRATE_SUPPORTED_LANGUAGES, getMigrateScanType } from '../lib/commands/migrate';
 import { RequireApproval } from '../lib/diff';
 import { availableInitLanguages, cliInit, printAvailableTemplates } from '../lib/init';
 import { data, debug, error, print, setLogLevel, setCI } from '../lib/logging';
@@ -281,6 +281,21 @@ async function parseCommandLineArguments(args: string[]) {
       .option('from-path', { type: 'string', desc: 'The path to the CloudFormation template to migrate. Use this for locally stored templates' })
       .option('from-stack', { type: 'boolean', desc: 'Use this flag to retrieve the template for an existing CloudFormation stack' })
       .option('output-path', { type: 'string', desc: 'The output path for the migrated CDK app' })
+      .option('from-scan', {
+        type: 'string',
+        desc: 'Determines if a new scan should be created, or the last successful existing scan should be used ' +
+          '\n options are "new" or "most-recent"',
+      })
+      .option('filter', {
+        type: 'array',
+        desc: 'Filters the resource scan based on the provided criteria in the following format: "key1=value1,key2=value2"' +
+          '\n This field can be passed multiple times for OR style filtering: ' +
+          '\n filtering options: ' +
+          '\n resource-identifier: A key-value pair that identifies the target resource. i.e. {"ClusterName", "myCluster"}' +
+          '\n resource-type-prefix: A string that represents a type-name prefix. i.e. "AWS::DynamoDB::"' +
+          '\n tag-key: a string that matches resources with at least one tag with the provided key. i.e. "myTagKey"' +
+          '\n tag-value: a string that matches resources with at least one tag with the provided value. i.e. "myTagValue"',
+      })
       .option('compress', { type: 'boolean', desc: 'Use this flag to zip the generated CDK app' }),
     )
     .command('context', 'Manage cached context values', (yargs: Argv) => yargs
@@ -679,6 +694,8 @@ export async function exec(args: string[], synthesizer?: Synthesizer): Promise<n
           fromStack: args['from-stack'],
           language: args.language,
           outputPath: args['output-path'],
+          fromScan: getMigrateScanType(args['from-scan']),
+          filter: args.filter,
           account: args.account,
           region: args.region,
           compress: args.compress,