ci: add support for e2e token rotation (#7665)

* ci: add support for e2e token rotation Currently we use long-lived, manually generated access tokens to set up AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables allowing our tests to run. This is a bad practice - authorization tokens should be automatically rotated to minimize risk of the tokens landing in the wrong hands. This PR sets up the Amplify CLI codebase to use temporary credentials during CI. Temporary STS credentials use an additional parameter, AWS_SESSION_TOKEN, to authorize the requests. This has been tested in CI on my fork and all tests pass: https://app.circleci.com/pipelines/github/johnpc/amplify-cli/144/workflows/fb12f075-d881-4c92-8cf7-f8647afb909e The token rotator itself is a lambda that runs every hour: https://github.com/johnpc/amplify-e2e-token-rotation-lambda * chore: use ini lib instead of manually updating file * chore: update injectSessionToken calls
aws-amplify · Jul 20, 2021 · 57c0a03 · 57c0a03
1 parent 6d16146
commit 57c0a03
Show file tree

Hide file tree

Showing 23 changed files with 68 additions and 39 deletions.
diff --git a/.circleci/config.base.yml b/.circleci/config.base.yml
@@ -17,10 +17,7 @@ node12: &node12
 defaults: &defaults
   working_directory: ~/repo
   docker:
-    - image: ${AWS_ECR_ACCOUNT_URL}/amplify-cli-e2e-base-image-repo:latest
-      aws_auth:
-        aws_access_key_id: $ECR_ACCESS_KEY
-        aws_secret_access_key: $ECR_SECRET_ACCESS_KEY
+    - image: ${AWS_ECR_ACCOUNT_URL}/amplify-cli-e2e-base-image-repo-public:latest
   resource_class: large
 
 clean_e2e_resources: &clean_e2e_resources

diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -16,10 +16,7 @@ node12:
 defaults:
   working_directory: ~/repo
   docker: &ref_1
-    - image: ${AWS_ECR_ACCOUNT_URL}/amplify-cli-e2e-base-image-repo:latest
-      aws_auth:
-        aws_access_key_id: $ECR_ACCESS_KEY
-        aws_secret_access_key: $ECR_SECRET_ACCESS_KEY
+    - image: ${AWS_ECR_ACCOUNT_URL}/amplify-cli-e2e-base-image-repo-public:latest
   resource_class: large
 clean_e2e_resources: &ref_7
   name: Cleanup resources

diff --git a/packages/amplify-console-integration-tests/src/profile-helper.ts b/packages/amplify-console-integration-tests/src/profile-helper.ts
@@ -35,6 +35,7 @@ export function getConfigFromProfile() {
   return {
     accessKeyId: credentials[profileName].aws_access_key_id,
     secretAccessKey: credentials[profileName].aws_secret_access_key,
+    sessionToken: credentials[profileName].aws_session_token,
     region: config[configKeyName].region,
   };
 }
@@ -62,15 +63,17 @@ export function setupAWSProfile() {
   Object.keys(credentials).forEach(key => {
     const keyName = key.trim();
     if (profileName === keyName) {
-      credentials[key].aws_access_key_id = process.env.CONSOLE_AWS_ACCESS_KEY_ID;
-      credentials[key].aws_secret_access_key = process.env.CONSOLE_AWS_SECRET_ACCESS_KEY;
+      credentials[key].aws_access_key_id = process.env.AWS_ACCESS_KEY_ID;
+      credentials[key].aws_secret_access_key = process.env.AWS_SECRET_ACCESS_KEY;
+      credentials[key].aws_session_token = process.env.AWS_SESSION_TOKEN;
       isCredSet = true;
     }
   });
   if (!isCredSet) {
     credentials[profileName] = {
-      aws_access_key_id: process.env.CONSOLE_AWS_ACCESS_KEY_ID,
-      aws_secret_access_key: process.env.CONSOLE_AWS_SECRET_ACCESS_KEY,
+      aws_access_key_id: process.env.AWS_ACCESS_KEY_ID,
+      aws_secret_access_key: process.env.AWS_SECRET_ACCESS_KEY,
+      aws_session_token: process.env.AWS_SESSION_TOKEN,
     };
   }
 

diff --git a/packages/amplify-e2e-core/src/index.ts b/packages/amplify-e2e-core/src/index.ts
@@ -1,8 +1,11 @@
 import * as os from 'os';
 import * as path from 'path';
 import * as fs from 'fs-extra';
+import * as ini from 'ini';
+
 import { spawnSync, execSync } from 'child_process';
 import { v4 as uuid } from 'uuid';
+import { pathManager } from 'amplify-cli-core';
 
 export * from './configure/';
 export * from './init/';
@@ -31,6 +34,13 @@ export function isCI(): boolean {
   return process.env.CI && process.env.CIRCLECI ? true : false;
 }
 
+export function injectSessionToken(profileName: string) {
+  const credentialsContents = ini.parse(fs.readFileSync(pathManager.getAWSCredentialsFilePath()).toString());
+  credentialsContents[profileName] = credentialsContents[profileName] || {};
+  credentialsContents[profileName].aws_session_token = process.env.AWS_SESSION_TOKEN;
+  fs.writeFileSync(pathManager.getAWSCredentialsFilePath(), ini.stringify(credentialsContents));
+}
+
 export function npmInstall(cwd: string) {
   spawnSync('npm', ['install'], { cwd });
 }

diff --git a/packages/amplify-e2e-core/src/init/initProjectHelper.ts b/packages/amplify-e2e-core/src/init/initProjectHelper.ts
@@ -380,7 +380,13 @@ export function amplifyInitYes(cwd: string): Promise<void> {
       env: {
         CLI_DEV_INTERNAL_DISABLE_AMPLIFY_APP_CREATION: '1',
       },
-    }).run((err: Error) => (err ? reject(err) : resolve()));
+    }).run((err: Error) =>
+      err
+        ? reject(err)
+        : (() => {
+            resolve();
+          })(),
+    );
   });
 }
 

diff --git a/packages/amplify-e2e-core/src/utils/envVars.ts b/packages/amplify-e2e-core/src/utils/envVars.ts
@@ -1,6 +1,7 @@
 type AWSCredentials = {
-  ACCESS_KEY_ID?: string;
-  SECRET_ACCESS_KEY?: string;
+  AWS_ACCESS_KEY_ID?: string;
+  AWS_SECRET_ACCESS_KEY?: string;
+  AWS_SESSION_TOKEN?: string;
 };
 type SocialProviders = {
   FACEBOOK_APP_ID?: string;

diff --git a/packages/amplify-e2e-core/src/utils/pinpoint.ts b/packages/amplify-e2e-core/src/utils/pinpoint.ts
@@ -14,6 +14,7 @@ const settings = {
   startCmd: '\r',
   accessKeyId: process.env.AWS_ACCESS_KEY_ID,
   secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+  sessionToken: process.env.AWS_SESSION_TOKEN,
   region: process.env.CLI_REGION,
   pinpointResourceName: 'testpinpoint',
 };
@@ -46,6 +47,7 @@ export async function pinpointAppExist(pinpointProjectId: string): Promise<boole
   const pinpointClient = new Pinpoint({
     accessKeyId: settings.accessKeyId,
     secretAccessKey: settings.secretAccessKey,
+    sessionToken: settings.sessionToken,
     region: _.get(serviceRegionMap, settings.region, defaultPinpointRegion),
   });
 

diff --git a/packages/amplify-e2e-tests/sample.env b/packages/amplify-e2e-tests/sample.env
@@ -1,10 +1,7 @@
 # Used for setting up a new profile
 AWS_ACCESS_KEY_ID=<your-access-key>
 AWS_SECRET_ACCESS_KEY=<your-secret-access-key>
-
-# Used for profile less init
-ACCESS_KEY_ID=<your-access-key>
-SECRET_ACCESS_KEY=<your-secret-access-key>
+AWS_SESSION_TOKEN=<optional-session-token>
 
 # Used for Auth Hosted UI
 FACEBOOK_APP_ID=<your-facebook-app-id>

diff --git a/packages/amplify-e2e-tests/src/__tests__/import_auth_1.test.ts b/packages/amplify-e2e-tests/src/__tests__/import_auth_1.test.ts
@@ -324,18 +324,18 @@ describe('auth import userpool only', () => {
     // Set it to make sure deleteProject error will be ignored
     ignoreProjectDeleteErrors = true;
 
-    const { ACCESS_KEY_ID, SECRET_ACCESS_KEY } = getEnvVars();
-    if (!ACCESS_KEY_ID || !SECRET_ACCESS_KEY) {
-      throw new Error('Set AWS_ACCESS_KEY_ID and SECRET_ACCESS_KEY either in .env file or as Environment variable');
+    const { AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY } = getEnvVars();
+    if (!AWS_ACCESS_KEY_ID || !AWS_SECRET_ACCESS_KEY) {
+      throw new Error('Set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY either in .env file or as Environment variable');
     }
 
     const newProjectRegion = process.env.CLI_REGION === 'us-west-2' ? 'us-east-2' : 'us-west-2';
 
     await initProjectWithAccessKey(projectRoot, {
       ...projectSettings,
       envName: 'integtest',
-      accessKeyId: ACCESS_KEY_ID,
-      secretAccessKey: SECRET_ACCESS_KEY,
+      accessKeyId: AWS_ACCESS_KEY_ID,
+      secretAccessKey: AWS_SECRET_ACCESS_KEY,
       region: newProjectRegion,
     } as any);
 

diff --git a/packages/amplify-e2e-tests/src/__tests__/init.test.ts b/packages/amplify-e2e-tests/src/__tests__/init.test.ts
@@ -83,13 +83,13 @@ describe('amplify init', () => {
   });
 
   it('should init project without profile', async () => {
-    const { ACCESS_KEY_ID, SECRET_ACCESS_KEY } = getEnvVars();
-    if (!ACCESS_KEY_ID || !SECRET_ACCESS_KEY) {
-      throw new Error('Set ACCESS_KEY_ID and SECRET_ACCESS_KEY either in .env file or as Environment variable');
+    const { AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY } = getEnvVars();
+    if (!AWS_ACCESS_KEY_ID || !AWS_SECRET_ACCESS_KEY) {
+      throw new Error('Set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY either in .env file or as Environment variable');
     }
     await initProjectWithAccessKey(projRoot, {
-      accessKeyId: ACCESS_KEY_ID,
-      secretAccessKey: SECRET_ACCESS_KEY,
+      accessKeyId: AWS_ACCESS_KEY_ID,
+      secretAccessKey: AWS_SECRET_ACCESS_KEY,
     });
 
     const meta = getProjectMeta(projRoot).providers.awscloudformation;
@@ -103,8 +103,8 @@ describe('amplify init', () => {
     // init new env
     await initNewEnvWithAccessKey(projRoot, {
       envName: 'foo',
-      accessKeyId: ACCESS_KEY_ID,
-      secretAccessKey: SECRET_ACCESS_KEY,
+      accessKeyId: AWS_ACCESS_KEY_ID,
+      secretAccessKey: AWS_SECRET_ACCESS_KEY,
     });
     const newEnvMeta = getProjectMeta(projRoot).providers.awscloudformation;
 

diff --git a/packages/amplify-e2e-tests/src/aws-matchers/iamMatcher.ts b/packages/amplify-e2e-tests/src/aws-matchers/iamMatcher.ts
@@ -36,6 +36,7 @@ export const toHaveValidPolicyConditionMatchingIdpId = async (roleName: string,
     const iam = new IAM({
       accessKeyId: process.env.AWS_ACCESS_KEY_ID,
       secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+      sessionToken: process.env.AWS_SESSION_TOKEN,
     });
 
     const { Role: role } = await iam.getRole({ RoleName: roleName }).promise();
@@ -55,9 +56,7 @@ export const toHaveValidPolicyConditionMatchingIdpId = async (roleName: string,
         return false;
       }
     });
-
     message = pass ? 'Found Matching Condition' : 'Matching Condition does not exist';
-
   } catch (e) {
     pass = false;
     message = 'IAM GetRole threw Error: ' + e.message;

diff --git a/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts b/packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts
@@ -79,6 +79,7 @@ const configureAws = (): void => {
     credentials: {
       accessKeyId: process.env.AWS_ACCESS_KEY_ID,
       secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+      sessionToken: process.env.AWS_SESSION_TOKEN,
       ...(process.env.AWS_SESSION_TOKEN ? { sessionToken: process.env.AWS_SESSION_TOKEN } : {}),
     },
     maxRetries: 10,

diff --git a/packages/amplify-e2e-tests/src/configure_tests.ts b/packages/amplify-e2e-tests/src/configure_tests.ts
@@ -1,4 +1,4 @@
-import { amplifyConfigure as configure, isCI } from 'amplify-e2e-core';
+import { amplifyConfigure as configure, injectSessionToken, isCI } from 'amplify-e2e-core';
 
 async function setupAmplify() {
   if (isCI()) {
@@ -14,6 +14,9 @@ async function setupAmplify() {
       profileName: 'amplify-integ-test-user',
       region: REGION,
     });
+    if (process.env.AWS_SESSION_TOKEN) {
+      injectSessionToken('amplify-integ-test-user');
+    }
   } else {
     console.log('AWS Profile is already configured');
   }

diff --git a/packages/amplify-e2e-tests/src/init-special-cases/index.ts b/packages/amplify-e2e-tests/src/init-special-cases/index.ts
@@ -7,6 +7,7 @@ export async function initWithoutCredentialFileAndNoNewUserSetup(projRoot) {
   const settings = {
     accessKeyId: process.env.AWS_ACCESS_KEY_ID,
     secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+    sessionToken: process.env.AWS_SESSION_TOKEN,
     region: process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION || 'us-west-2',
   };
 

diff --git a/packages/amplify-e2e-tests/src/schema-api-directives/authHelper.ts b/packages/amplify-e2e-tests/src/schema-api-directives/authHelper.ts
@@ -70,6 +70,7 @@ export function getConfiguredCognitoClient(): CognitoIdentityServiceProvider {
   const awsconfig = {
     accessKeyId: process.env.AWS_ACCESS_KEY_ID,
     secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+    sessionToken: process.env.AWS_SESSION_TOKEN,
     region: process.env.CLI_REGION,
   };
 
@@ -124,6 +125,7 @@ export function getConfiguredAppsyncClientIAMAuth(url: string, region: string):
       credentials: {
         accessKeyId: process.env.AWS_ACCESS_KEY_ID,
         secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+        sessionToken: process.env.AWS_SESSION_TOKEN,
       },
     },
   });

diff --git a/packages/amplify-e2e-tests/src/schema-api-directives/tests/predictions-usage.ts b/packages/amplify-e2e-tests/src/schema-api-directives/tests/predictions-usage.ts
@@ -45,6 +45,7 @@ async function uploadImageFile(projectDir: string) {
   const s3Client = new aws.S3({
     accessKeyId: process.env.AWS_ACCESS_KEY_ID,
     secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+    sessionToken: process.env.AWS_SESSION_TOKEN,
     region: process.env.AWS_DEFAULT_REGION,
   });
 

diff --git a/packages/amplify-migration-tests/sample.env b/packages/amplify-migration-tests/sample.env
@@ -1,10 +1,7 @@
 # Used for setting up a new profile
 AWS_ACCESS_KEY_ID=<your-access-key>
 AWS_SECRET_ACCESS_KEY=<your-secret-access-key>
-
-# Used for profile less init
-ACCESS_KEY_ID=<your-access-key>
-SECRET_ACCESS_KEY=<your-secret-access-key>
+AWS_SESSION_TOKEN=<optional-session-token>
 
 # Used for Auth Hosted UI
 FACEBOOK_APP_ID=<your-facebook-app-id>

diff --git a/packages/amplify-migration-tests/src/configure_tests.ts b/packages/amplify-migration-tests/src/configure_tests.ts
@@ -1,4 +1,4 @@
-import { amplifyConfigure as configure, isCI, installAmplifyCLI } from 'amplify-e2e-core';
+import { amplifyConfigure as configure, isCI, installAmplifyCLI, injectSessionToken } from 'amplify-e2e-core';
 
 /*
  *  Migration tests must be run without publishing to local registry
@@ -21,6 +21,9 @@ async function setupAmplify(version: string = 'latest') {
       secretAccessKey: AWS_SECRET_ACCESS_KEY,
       profileName: 'amplify-integ-test-user',
     });
+    if (process.env.AWS_SESSION_TOKEN) {
+      injectSessionToken('amplify-integ-test-user');
+    }
   } else {
     console.log('AWS Profile is already configured');
   }

diff --git a/packages/amplify-provider-awscloudformation/src/configuration-manager.ts b/packages/amplify-provider-awscloudformation/src/configuration-manager.ts
@@ -217,6 +217,7 @@ async function initialize(context: $TSContext, authConfig?: AuthFlowConfig) {
     ) {
       awsConfigInfo.config.accessKeyId = awsConfigInfo.config.accessKeyId || authConfig.accessKeyId;
       awsConfigInfo.config.secretAccessKey = awsConfigInfo.config.secretAccessKey || authConfig.secretAccessKey;
+      awsConfigInfo.config.sessionToken = awsConfigInfo.config.sessionToken || authConfig.sessionToken;
       awsConfigInfo.config.region = awsConfigInfo.config.region || authConfig.region;
     } else {
       await promptForAuthConfig(context, authConfig);
@@ -430,6 +431,7 @@ async function promptForAuthConfig(context: $TSContext, authConfig?: AuthFlowCon
   if (!obfuscateUtil.isObfuscated(answers.secretAccessKey)) {
     awsConfigInfo.config.secretAccessKey = answers.secretAccessKey;
   }
+  awsConfigInfo.config.sessionToken = awsConfigInfo.config.sessionToken || process.env.AWS_SESSION_TOKEN;
   awsConfigInfo.config.region = answers.region;
 }
 
@@ -455,6 +457,7 @@ async function validateConfig(context: $TSContext) {
         credentials: {
           accessKeyId: awsConfigInfo.config.accessKeyId,
           secretAccessKey: awsConfigInfo.config.secretAccessKey,
+          sessionToken: awsConfigInfo.config.sessionToken,
         },
       });
       try {
@@ -495,6 +498,7 @@ function persistLocalEnvConfig(context: $TSContext) {
       const awsSecrets = {
         accessKeyId: awsConfigInfo.config.accessKeyId,
         secretAccessKey: awsConfigInfo.config.secretAccessKey,
+        sessionToken: awsConfigInfo.config.sessionToken,
         region: awsConfigInfo.config.region,
       };
       const sharedConfigDirPath = path.join(pathManager.getHomeDotAmplifyDirPath(), constants.ProviderName);
@@ -774,6 +778,7 @@ export async function getAwsConfig(context: $TSContext): Promise<AwsSdkConfig> {
       resultAWSConfigInfo = {
         accessKeyId: awsConfigInfo.config.accessKeyId,
         secretAccessKey: awsConfigInfo.config.secretAccessKey,
+        sessionToken: awsConfigInfo.config.sessionToken,
         region: awsConfigInfo.config.region,
       };
     }

diff --git a/packages/amplify-provider-awscloudformation/src/setup-new-user.js b/packages/amplify-provider-awscloudformation/src/setup-new-user.js
@@ -12,6 +12,7 @@ async function run(context) {
   const awsConfigInfo = {
     accessKeyId: constants.DefaultAWSAccessKeyId,
     secretAccessKey: constants.DefaultAWSSecretAccessKey,
+    sessionToken: process.env.AWS_SESSION_TOKEN,
     region: constants.DefaultAWSRegion,
   };
 

diff --git a/packages/amplify-util-mock/src/utils/dynamo-db/index.ts b/packages/amplify-util-mock/src/utils/dynamo-db/index.ts
@@ -42,6 +42,7 @@ export function configureDDBDataSource(config, ddbConfig) {
           region: ddbConfig.region,
           accessKeyId: ddbConfig.accessKeyId,
           secretAccessKey: ddbConfig.secretAccessKey,
+          sessionToken: ddbConfig.sessionToken || process.env.AWS_SESSION_TOKEN,
         },
       };
     }),

diff --git a/...es/graphql-transformers-e2e-tests/src/__tests__/MultiAuthModelAuthTransformer.e2e.test.ts b/...es/graphql-transformers-e2e-tests/src/__tests__/MultiAuthModelAuthTransformer.e2e.test.ts
@@ -548,6 +548,7 @@ beforeAll(async () => {
         credentials: {
           accessKeyId: unauthCredentials.accessKeyId,
           secretAccessKey: unauthCredentials.secretAccessKey,
+          sessionToken: unauthCredentials.sessionToken,
         },
       },
       offlineConfig: {

diff --git a/packages/graphql-transformers-e2e-tests/src/__tests__/NonModelAuthFunction.e2e.test.ts b/packages/graphql-transformers-e2e-tests/src/__tests__/NonModelAuthFunction.e2e.test.ts
@@ -401,6 +401,7 @@ beforeAll(async () => {
       credentials: {
         accessKeyId: unauthCredentials.accessKeyId,
         secretAccessKey: unauthCredentials.secretAccessKey,
+        sessionToken: unauthCredentials.sessionToken,
       },
     },
     offlineConfig: {