[AAE-4608] Use smartrunner version 2 and add SuperCache (workspaces +…

… S3 combo)! (#6704) * [AAE-4608] Use smartrunner version 2 * Fix npm installing and do not depend on develop cache * Add Travis workspaces the simplest way * Add missing dependencies * Remove unnecessary npm install on lint * Fix travis indentation * Fix e2e missing adf packages * Trying to fix Travis * Remove debug information * Use Travis workspaces for smartrunner and built artefacts caching instead of S3 * Implement nuclear caching for node_modules * Quiet snyk running, since the log length kills Travis * Maybe with -q * Quieting snyk * Why displaying the added packages?????? * Fix S3 persister/retriever scripts * Restore back parallel build and lint, to gain 10 minutes
Alfresco · Feb 22, 2021 · f84ea5d · f84ea5d
1 parent f6308a7
commit f84ea5d
Show file tree

Hide file tree

Showing 13 changed files with 436 additions and 266 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -9,11 +9,15 @@ node_js:
   - '12.18.4'
 
 before_install: . ./scripts/ci/job_hooks/before_install.sh
-install:
-    echo "no install"
+install: ./scripts/ci/job_hooks/install.sh
 
 env:
     global:
+        - S3_DBP_PATH="s3://alfresco-travis-builds/adf"
+        - DEMO_SHELL_DIR="./dist/demo-shell"
+        - BUILT_LIBS_DIR="./lib/dist"
+        - NODE_MODULES_DIR="./node_modules"
+        - SMART_RUNNER_DIRECTORY=".protractor-smartrunner"
         #E2E VARIABLES
         - SAVE_SCREENSHOT=true
         - REDIRECT_URI=/
@@ -30,7 +34,8 @@ branches:
   - /.*beta.*/
 
 stages:
-  - name: Lint & Build Dist & Release
+  - name: Setup
+  - name: "Lint & Build Dist & Release"
     if: tag IS blank
   - name: Check bundle
     if: type = push AND tag IS blank
@@ -54,43 +59,73 @@ services:
 
 jobs:
     include:
+        # Setup ======================
+        - stage: Setup
+          name: "Node modules cache preparation"
+          script: echo "Only here for setting up the shared node_modules"
+          workspaces:
+            create:
+              name: node_modules_cache
+              paths:
+                - "$NODE_MODULES_DIR"
+            use: node_modules_cache
+
         # Run Only for any PR
-        - stage: Lint & Build Dist & Release
+        - stage: "Lint & Build Dist & Release"
           name:  Build
           script:
             - ./scripts/travis/build/build.sh
             - ./scripts/travis/release/release-npm.sh
             - ./scripts/travis/release/release-docker.sh
-
-          after_success:
-            - ./scripts/ci/utils/artifact-to-s3.sh -a ./dist/demo-shell -o "$S3_DBP_FOLDER/alfresco-demoshell.tar.bz2"
-            - ./scripts/ci/utils/artifact-to-s3.sh -a ./lib/dist -o "$S3_DBP_FOLDER/alfresco-libs.tar.bz2"
-
-        - stage: Lint & Build Dist & Release
-          name:  Lint
-          script:
-             - ./scripts/lint.sh
+          workspaces:
+            create:
+              name: built_artefacts_cache
+              paths:
+                - "$DEMO_SHELL_DIR"
+                - "$BUILT_LIBS_DIR"
+            use: node_modules_cache
+
+        - stage: "Lint & Build Dist & Release"
+          name: "Lint"
+          script: ./scripts/lint.sh
+          workspaces:
+            use: node_modules_cache
 
         - stage: Unit test
           name: Unit test content
           script:  ./scripts/travis/unit-test/content.sh
+          workspaces:
+            use: node_modules_cache
+
         - stage: Unit test
           name: Unit test core extension demo
           script: ./scripts/travis/unit-test/core-extension.sh
+          workspaces:
+            use: node_modules_cache
+
         - stage: Unit test
           name: Unit test process insights
           script: ./scripts/travis/unit-test/process.sh
+          workspaces:
+            use: node_modules_cache
+
         - stage: Unit test
           name: Unit test process-cloud
           script: ./scripts/travis/unit-test/process-cloud.sh
+          workspaces:
+            use: node_modules_cache
 
         - stage: Trigger Alpha ADF child build
           name: Trigger Alpha ADF child build
           script: ./scripts/travis/update/update-project.sh -p $TRAVIS_BUILD_NUMBER -t $GITHUB_TOKEN -v alpha
+          workspaces:
+            use: node_modules_cache
 
         - stage: Trigger Beta ADF child build
           name: Trigger Beta ADF child build
           script: ./scripts/travis/update/update-project.sh -p $TRAVIS_BUILD_NUMBER -t $GITHUB_TOKEN -v beta
+          workspaces:
+            use: node_modules_cache
 
         - stage: Release tag
           script: ./scripts/travis/release/git-tag.sh
@@ -103,6 +138,15 @@ jobs:
             - ./scripts/ci/check-env/check-ps-env.sh || travis_terminate 1
           script: ./scripts/travis/e2e/core-e2e.sh
           after_script: ./scripts/ci/job_hooks/after_e2e.sh
+          workspaces:
+            create:
+              name: smartrunner-cache-core
+              paths:
+                - "$SMART_RUNNER_DIRECTORY"
+            use:
+              - node_modules_cache
+              - built_artefacts_cache
+              - smartrunner-cache-core
 
         - stage: e2e Test
           name:  content
@@ -111,6 +155,15 @@ jobs:
               - ./scripts/ci/check-env/check-cs-env.sh || travis_terminate 1
           script: ./scripts/travis/e2e/content-services-e2e.sh
           after_script: ./scripts/ci/job_hooks/after_e2e.sh
+          workspaces:
+            create:
+              name: smartrunner-cache-content
+              paths:
+                - "$SMART_RUNNER_DIRECTORY"
+            use:
+              - node_modules_cache
+              - built_artefacts_cache
+              - smartrunner-cache-content
 
         - stage: e2e Test
           name:  search
@@ -119,6 +172,15 @@ jobs:
               - ./scripts/ci/check-env/check-cs-env.sh || travis_terminate 1
           script: ./scripts/travis/e2e/search-e2e.sh
           after_script: ./scripts/ci/job_hooks/after_e2e.sh
+          workspaces:
+            create:
+              name: smartrunner-cache-search
+              paths:
+                - "$SMART_RUNNER_DIRECTORY"
+            use:
+              - node_modules_cache
+              - built_artefacts_cache
+              - smartrunner-cache-search
 
         - stage: e2e Test
           name: process
@@ -128,6 +190,15 @@ jobs:
               - ./scripts/ci/check-env/check-external-cs-env.sh || travis_terminate 1
           script: ./scripts/travis/e2e/process-services-e2e.sh
           after_script: ./scripts/ci/job_hooks/after_e2e.sh
+          workspaces:
+            create:
+              name: smartrunner-cache-process
+              paths:
+                - "$SMART_RUNNER_DIRECTORY"
+            use:
+              - node_modules_cache
+              - built_artefacts_cache
+              - smartrunner-cache-process
 
         - stage: e2e Test
           name: process Cloud
@@ -137,11 +208,22 @@ jobs:
               - ./scripts/ci/check-env/check-ps-cloud-env.sh || travis_terminate 1
           script: ./scripts/travis/e2e/process-services-cloud-e2e.sh
           after_script: ./scripts/ci/job_hooks/after_e2e.sh
+          workspaces:
+            create:
+              name: smartrunner-cache-process-cloud
+              paths:
+                - "$SMART_RUNNER_DIRECTORY"
+            use:
+              - node_modules_cache
+              - built_artefacts_cache
+              - smartrunner-cache-process-cloud
 
         - stage: Check bundle
           script:
           - ADF_VERSION=$(npm view @alfresco/adf-core@${TAG_NPM} version)
           - ./scripts/travis/build/npm-check-bundles.sh -v ${ADF_VERSION}
+          workspaces:
+            use: node_modules_cache
 
 notifications:
   slack:
@@ -156,6 +238,6 @@ notifications:
       - "Message: %{message}"
 
 cache:
+  npm: true
   directories:
-  - node_modules
   - tmp
diff --git a/e2e/protractor.conf.js b/e2e/protractor.conf.js
@@ -1,13 +1,13 @@
 const {LocalStorageUtil, Logger} = require('@alfresco/adf-testing');
 const path = require('path');
 const {SpecReporter} = require('jasmine-spec-reporter');
-const retry = require('protractor-retry').retry;
+const retry = require('protractor-retry-angular-cli').retry;
 const tsConfig = require('./tsconfig.e2e.json');
 const testConfig = require('./test.config');
 const RESOURCES = require('./util/resources');
-const SmartRunner = require('protractor-smartrunner');
 const resolve = require('path').resolve;
 const fs = require('fs');
+const smartRunnerFactory = require('./smartrunner-factory');
 
 const {uploadScreenshot} = require('./protractor/save-remote');
 const argv = require('yargs').argv;
@@ -166,9 +166,7 @@ exports.config = {
         includeStackTrace: true,
         print: () => {
         },
-        ...SmartRunner.withOptionalExclusions(
-            resolve(__dirname, './protractor.excludes.json')
-        )
+        ...(process.env.CI ? smartRunnerFactory.applyExclusionFilter() : {} )
     },
 
     /**
@@ -200,11 +198,7 @@ exports.config = {
     async onPrepare() {
         if (process.env.CI) {
             retry.onPrepare();
-            const repoHash = process.env.GIT_HASH || '';
-            const outputDirectory = process.env.SMART_RUNNER_DIRECTORY;
-            console.log(`SmartRunner's repoHash: "${repoHash}"`);
-            console.log(`SmartRunner's outputDirectory: "${outputDirectory}"`);
-            SmartRunner.apply({outputDirectory, repoHash});
+            smartRunnerFactory.getInstance().onPrepare();
         }
 
         jasmine.DEFAULT_TIMEOUT_INTERVAL = TIMEOUT;
@@ -282,7 +276,7 @@ exports.config = {
 
     },
 
-    afterLaunch: async function () {
+    afterLaunch: async function (statusCode) {
         if (SAVE_SCREENSHOT) {
             console.log(`Save screenshot enabled`);
 
@@ -299,7 +293,7 @@ exports.config = {
             console.log(`Save screenshot disabled`);
         }
 
-        return retry.afterLaunch(MAX_RETRIES);
+        return retry.afterLaunch(MAX_RETRIES, statusCode);
     }
 
 };
diff --git a/e2e/smartrunner-factory.js b/e2e/smartrunner-factory.js
@@ -0,0 +1,14 @@
+const SmartRunnerFactory = require('protractor-smartrunner').SmartRunnerFactory;
+const resolve = require('path').resolve;
+
+const outputDirectory = process.env.SMART_RUNNER_DIRECTORY;
+const repoHash = process.env.GIT_HASH;
+
+console.log(`SmartRunner's repoHash: "${repoHash}"`);
+console.log(`SmartRunner's outputDirectory: "${outputDirectory}"`);
+
+module.exports = new SmartRunnerFactory({
+    repoHash,
+    ...(outputDirectory ? { outputDirectory: resolve(__dirname, '..', outputDirectory) } : {}),
+    exclusionPath: resolve(__dirname, 'protractor.excludes.json')
+});