ci: misc updates to circleci config

* test: fix e2e amplify dir

* test: update api default testingWithLatestCodebase

* ci: shorten test suite names

* fix test names in split-e2e-tests script

* skip api_5 on windows

.circleci/config.base.yml

@@ -6,7 +6,7 @@ machine:
     PATH: '${PATH}:${HOME}/${CIRCLE_PROJECT_REPONAME}/node_modules/.bin'
 
 executors:
-  windows: &windows-e2e-executor
+  w: &windows-e2e-executor
     machine:
       image: 'windows-server-2019-vs2019:stable'
       resource_class: 'windows.large'
@@ -16,7 +16,7 @@ executors:
       AMPLIFY_DIR: C:/home/circleci/repo/out
       AMPLIFY_PATH: C:/home/circleci/repo/out/amplify.exe
 
-  linux: &linux-e2e-executor
+  l: &linux-e2e-executor
     docker:
       - image: public.ecr.aws/a6e6w2n0/amplify-cli-e2e-base-image-repo-public:latest
     working_directory: ~/repo
@@ -30,7 +30,7 @@ defaults: &defaults
   parameters:
     os:
       type: executor
-      default: linux
+      default: l
   executor: << parameters.os >>
 
 clean_e2e_resources: &clean_e2e_resources
@@ -67,7 +67,7 @@ jobs:
     parameters:
       os:
         type: executor
-        default: linux
+        default: l
     executor: << parameters.os >>
     steps:
       - checkout
@@ -261,7 +261,7 @@ jobs:
     parameters:
       os:
         type: executor
-        default: os.linux
+        default: l
     executor: << parameters.os >>
     working_directory: ~/repo
     steps:
@@ -704,8 +704,8 @@ workflows:
           matrix:
             parameters:
               os:
-                - linux
-                - windows
+                - l
+                - w
       - test:
           requires:
             - build
@@ -955,7 +955,7 @@ commands:
     parameters:
       os:
         type: executor
-        default: linux-e2e-executor
+        default: l
     steps:
       - when:
           condition:
@@ -998,7 +998,7 @@ commands:
     parameters:
       os:
         type: executor
-        default: linux-e2e-executor
+        default: l
     steps:
       - when:
           condition:
@@ -1016,7 +1016,7 @@ commands:
     parameters:
       os:
         type: executor
-        default: linux-e2e-executor
+        default: l
     steps:
       - when:
           condition:
@@ -1032,7 +1032,7 @@ commands:
     parameters:
       os:
         type: executor
-        default: linux-e2e-executor
+        default: l
     steps:
       - when:
           condition:
@@ -1063,7 +1063,7 @@ commands:
     parameters:
       os:
         type: executor
-        default: linux-e2e-executor
+        default: l
     steps:
       - run:
           name: Scan E2E artifacts
@@ -1079,7 +1079,7 @@ commands:
     parameters:
       os:
         type: executor
-        default: linux-e2e-executor
+        default: l
     steps:
       - run:
           name: Clean job resources

packages/amplify-e2e-core/src/categories/api.ts

@@ -32,7 +32,7 @@ interface AddApiOptions {
 
 const defaultOptions: AddApiOptions = {
   apiName: '\r',
-  testingWithLatestCodebase: true,
+  testingWithLatestCodebase: false,
 };
 
 export function addApiWithoutSchema(cwd: string, opts: Partial<AddApiOptions & { apiKeyExpirationDays: number }> = {}) {

scripts/split-e2e-tests.ts

@@ -11,67 +11,68 @@ const CONCURRENCY = 25;
 // Each of these failures should be independently investigated, resolved, and removed from this list.
 // For now, this list is being used to skip creation of circleci jobs for these tasks
 const WINDOWS_TEST_FAILURES = [
-  'api_6-amplify_e2e_tests',
-  'datastore-modelgen-amplify_e2e_tests',
-  'delete-amplify_e2e_tests',
-  'env-amplify_e2e_tests',
-  'feature-flags-amplify_e2e_tests',
-  'function_1-amplify_e2e_tests',
-  'function_2-amplify_e2e_tests',
-  'function_3-amplify_e2e_tests',
-  'function_4-amplify_e2e_tests',
-  'function_5-amplify_e2e_tests',
-  'function_6-amplify_e2e_tests',
-  'function_7-amplify_e2e_tests',
-  'function_8-amplify_e2e_tests',
-  'function_9-amplify_e2e_tests',
-  'geo-remove-amplify_e2e_tests',
-  'geo-update-amplify_e2e_tests',
-  'layer-1-amplify_e2e_tests',
-  'layer-2-amplify_e2e_tests',
-  'layer-3-amplify_e2e_tests',
-  'layer-4-amplify_e2e_tests',
-  'migration-api-connection-migration-amplify_e2e_tests',
-  'migration-api-connection-migration2-amplify_e2e_tests',
-  'migration-api-key-migration1-amplify_e2e_tests',
-  'migration-api-key-migration2-amplify_e2e_tests',
-  'pull-amplify_e2e_tests',
-  'schema-iterative-rollback-1-amplify_e2e_tests',
-  'schema-iterative-rollback-2-amplify_e2e_tests',
-  'schema-iterative-update-1-amplify_e2e_tests',
-  'schema-iterative-update-2-amplify_e2e_tests',
-  'schema-iterative-update-3-amplify_e2e_tests',
-  'schema-iterative-update-4-amplify_e2e_tests',
-  'schema-iterative-update-locking-amplify_e2e_tests',
-  'schema-key-amplify_e2e_tests',
-  'api_4-amplify_e2e_tests',
-  'api_3-amplify_e2e_tests',
-  'api_2-amplify_e2e_tests',
-  'api_1-amplify_e2e_tests',
-  'amplify-app-amplify_e2e_tests',
-  'import_s3_1-amplify_e2e_tests',
-  'import_s3_3-amplify_e2e_tests',
-  'auth_4-amplify_e2e_tests',
-  'auth_3-amplify_e2e_tests',
-  'custom_policies_container-amplify_e2e_tests',
-  'custom_policies_function-amplify_e2e_tests',
-  'storage-4-amplify_e2e_tests',
-  'resolvers-amplify_e2e_tests',
-  'migration-api-key-migration3-amplify_e2e_tests',
-  'migration-api-key-migration4-amplify_e2e_tests',
-  'migration-api-key-migration5-amplify_e2e_tests',
-  'transformer-migrations-model-migration-amplify_e2e_tests',
-  'global_sandbox-amplify_e2e_tests',
-  'api_5-amplify_e2e_tests',
+  'api_6_pkg',
+  'datastore-modelgen_pkg',
+  'delete_pkg',
+  'env_pkg',
+  'feature-flags_pkg',
+  'function_1_pkg',
+  'function_2_pkg',
+  'function_3_pkg',
+  'function_4_pkg',
+  'function_5_pkg',
+  'function_6_pkg',
+  'function_7_pkg',
+  'function_8_pkg',
+  'function_9_pkg',
+  'geo-remove_pkg',
+  'geo-update_pkg',
+  'layer-1_pkg',
+  'layer-2_pkg',
+  'layer-3_pkg',
+  'layer-4_pkg',
+  'api-connection-migration_pkg',
+  'api-connection-migration2_pkg',
+  'api-key-migration1_pkg',
+  'api-key-migration2_pkg',
+  'pull_pkg',
+  'schema-iterative-rollback-1_pkg',
+  'schema-iterative-rollback-2_pkg',
+  'schema-iterative-update-1_pkg',
+  'schema-iterative-update-2_pkg',
+  'schema-iterative-update-3_pkg',
+  'schema-iterative-update-4_pkg',
+  'schema-iterative-update-locking_pkg',
+  'schema-key_pkg',
+  'api_5_pkg',
+  'api_4_pkg',
+  'api_3_pkg',
+  'api_2_pkg',
+  'api_1_pkg',
+  'amplify-app_pkg',
+  'import_s3_1_pkg',
+  'import_s3_3_pkg',
+  'auth_4_pkg',
+  'auth_3_pkg',
+  'custom_policies_container_pkg',
+  'custom_policies_function_pkg',
+  'storage-4_pkg',
+  'resolvers_pkg',
+  'api-key-migration2_pkg',
+  'api-key-migration3_pkg',
+  'api-key-migration4_pkg',
+  'api-key-migration5_pkg',
+  'model-migration_pkg',
+  'global_sandbox_pkg',
 
   // 👇 These fail due to ExpiredToken. 👇
   // 👇 Tests should be split to speed up execution time. 👇
-  'geo-add-amplify_e2e_tests',
-  'import_auth_1-amplify_e2e_tests',
-  'import_auth_2-amplify_e2e_tests',
-  'import_auth_3-amplify_e2e_tests',
-  'import_dynamodb_2-amplify_e2e_tests',
-  'import_s3_2-amplify_e2e_tests',
+  'geo-add_pkg',
+  'import_auth_1_pkg',
+  'import_auth_2_pkg',
+  'import_auth_3_pkg',
+  'import_dynamodb_2_pkg',
+  'import_s3_2_pkg',
 ];
 
 // Ensure to update packages/amplify-e2e-tests/src/cleanup-e2e-resources.ts is also updated this gets updated
@@ -87,19 +88,19 @@ const AWS_REGIONS_TO_RUN_TESTS = [
 
 // Some services (eg. amazon lex) are not available in all regions
 // Tests added to this list will always run in us-west-2
-const FORCE_US_WEST_2 = ['interactions-amplify_e2e_tests'];
+const FORCE_US_WEST_2 = ['interactions'];
 
 const USE_PARENT_ACCOUNT = [
-  'api_2-amplify_e2e_tests',
-  'api_1-amplify_e2e_tests',
-  'auth_2-amplify_e2e_tests',
-  'import_dynamodb_1-amplify_e2e_tests',
-  'import_s3_1-amplify_e2e_tests',
-  'migration-api-key-migration2-amplify_e2e_tests',
-  'migration-api-key-migration3-amplify_e2e_tests',
-  'migration-api-key-migration4-amplify_e2e_tests',
-  'migration-api-key-migration5-amplify_e2e_tests',
-  'storage-amplify_e2e_tests',
+  'api_2',
+  'api_1',
+  'auth_2',
+  'import_dynamodb_1',
+  'import_s3_1',
+  'api-key-migration2',
+  'api-key-migration3',
+  'api-key-migration4',
+  'api-key-migration5',
+  'storage',
 ];
 
 // This array needs to be update periodically when new tests suites get added
@@ -230,12 +231,17 @@ function getTestFiles(dir: string, pattern = 'src/**/*.test.ts'): string[] {
 }
 
 function generateJobName(baseName: string, testSuitePath: string): string {
-  return `${testSuitePath
-    .replace('src/', '')
-    .replace('__tests__/', '')
-    .replace(/test\.ts$/, '')
-    .replace(/\//g, '-')
-    .replace(/\./g, '-')}${baseName}`;
+  const startIndex = testSuitePath.lastIndexOf('/') + 1;
+  const endIndex = testSuitePath.lastIndexOf('.test');
+  let name = testSuitePath.substring(startIndex, endIndex).split('.e2e').join('').split('.').join('-');
+  if (baseName.includes('pkg')) {
+    name = name + '_pkg';
+  }
+  if (baseName.includes('amplify_migration_tests')) {
+    const startIndex = baseName.lastIndexOf('_');
+    name = name + baseName.substring(startIndex);
+  }
+  return name;
 }
 
 /**
@@ -276,9 +282,9 @@ function splitTests(
     const isPkg = newJobName.endsWith('_pkg');
     if (!isPkg) {
       (newJob.environment as any) = {
+        ...newJob.environment,
         AMPLIFY_DIR: '/home/circleci/repo/packages/amplify-cli/bin',
         AMPLIFY_PATH: '/home/circleci/repo/packages/amplify-cli/bin/amplify',
-        ...newJob.environment,
       };
     }
     return { ...acc, [newJobName]: newJob };
@@ -322,8 +328,8 @@ function splitTests(
                   parameters: {
                     os:
                       WINDOWS_TEST_FAILURES.some(failingJob => newJobName.startsWith(failingJob)) || !newJobName.endsWith('_pkg')
-                        ? ['linux']
-                        : ['linux', 'windows'],
+                        ? ['l']
+                        : ['l', 'w'],
                   },
                 },
               },