From 11e793257f84ef19c26a59e71c1841b4cf4f41a8 Mon Sep 17 00:00:00 2001 From: Rafal Szmit <40626242+rafalszmit@users.noreply.github.com> Date: Wed, 17 Aug 2022 11:39:28 +0200 Subject: [PATCH] [AAE-10261] SmartRunner fix (#7760) * [AAE-1026] s3 store + fake test * [ci:force] test run 1 * [ci:force] test run 2 * [ci:force] test run 3 - one job * [ci:force] path fix * [ci:force] install awscli * add after script for all jobs * [ci:force] trigger travis * remove fake test * trigger travis --- .travis.yml | 16 +++++++++++ scripts/ci/job_hooks/after_e2e.sh | 5 ++++ scripts/ci/job_hooks/before_e2e.sh | 3 ++ scripts/ci/job_hooks/before_install.sh | 3 ++ scripts/ci/utils/artifact-from-s3.sh | 38 ++++++++++++++++++++++++++ scripts/ci/utils/artifact-to-s3.sh | 27 ++++++++++++++++++ 6 files changed, 92 insertions(+) create mode 100755 scripts/ci/job_hooks/after_e2e.sh create mode 100755 scripts/ci/utils/artifact-from-s3.sh create mode 100755 scripts/ci/utils/artifact-to-s3.sh diff --git a/.travis.yml b/.travis.yml index 623f182be11..4e8a31587bf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -197,6 +197,7 @@ jobs: - ./scripts/ci/check-env/check-cs-env.sh || travis_terminate 1 - ./scripts/ci/check-env/check-ps-env.sh || travis_terminate 1 script: ./scripts/travis/e2e/e2e.sh + after_script: ./scripts/ci/job_hooks/after_e2e.sh workspaces: create: name: e2e_cache @@ -217,6 +218,7 @@ jobs: - ./scripts/ci/job_hooks/before_e2e.sh - ./scripts/ci/check-env/check-cs-env.sh || travis_terminate 1 script: ./scripts/travis/e2e/e2e.sh + after_script: ./scripts/ci/job_hooks/after_e2e.sh workspaces: create: name: e2e_cache @@ -237,6 +239,7 @@ jobs: - ./scripts/ci/job_hooks/before_e2e.sh - ./scripts/ci/check-env/check-cs-env.sh || travis_terminate 1 script: ./scripts/travis/e2e/e2e.sh + after_script: ./scripts/ci/job_hooks/after_e2e.sh workspaces: create: name: e2e_cache @@ -257,6 +260,7 @@ jobs: - ./scripts/ci/job_hooks/before_e2e.sh - ./scripts/ci/check-env/check-cs-env.sh || travis_terminate 1 script: ./scripts/travis/e2e/e2e.sh + after_script: ./scripts/ci/job_hooks/after_e2e.sh workspaces: create: name: e2e_cache @@ -277,6 +281,7 @@ jobs: - ./scripts/ci/job_hooks/before_e2e.sh - ./scripts/ci/check-env/check-cs-env.sh || travis_terminate 1 script: ./scripts/travis/e2e/e2e.sh + after_script: ./scripts/ci/job_hooks/after_e2e.sh workspaces: create: name: e2e_cache @@ -297,6 +302,7 @@ jobs: - ./scripts/ci/job_hooks/before_e2e.sh - ./scripts/ci/check-env/check-cs-env.sh || travis_terminate 1 script: ./scripts/travis/e2e/e2e.sh + after_script: ./scripts/ci/job_hooks/after_e2e.sh workspaces: create: name: e2e_cache @@ -317,6 +323,7 @@ jobs: - ./scripts/ci/job_hooks/before_e2e.sh - ./scripts/ci/check-env/check-cs-env.sh || travis_terminate 1 script: ./scripts/travis/e2e/e2e.sh + after_script: ./scripts/ci/job_hooks/after_e2e.sh workspaces: create: name: e2e_cache @@ -338,6 +345,7 @@ jobs: - ./scripts/ci/check-env/check-ps-env.sh || travis_terminate 1 - ./scripts/ci/check-env/check-external-cs-env.sh || travis_terminate 1 script: ./scripts/travis/e2e/e2e.sh + after_script: ./scripts/ci/job_hooks/after_e2e.sh workspaces: create: name: e2e_cache @@ -359,6 +367,7 @@ jobs: - ./scripts/ci/check-env/check-ps-env.sh || travis_terminate 1 - ./scripts/ci/check-env/check-external-cs-env.sh || travis_terminate 1 script: ./scripts/travis/e2e/e2e.sh + after_script: ./scripts/ci/job_hooks/after_e2e.sh workspaces: create: name: e2e_cache @@ -380,6 +389,7 @@ jobs: - ./scripts/ci/check-env/check-ps-env.sh || travis_terminate 1 - ./scripts/ci/check-env/check-external-cs-env.sh || travis_terminate 1 script: ./scripts/travis/e2e/e2e.sh + after_script: ./scripts/ci/job_hooks/after_e2e.sh workspaces: create: name: e2e_cache @@ -401,6 +411,7 @@ jobs: - ./scripts/ci/check-env/check-ps-env.sh || travis_terminate 1 - ./scripts/ci/check-env/check-external-cs-env.sh || travis_terminate 1 script: ./scripts/travis/e2e/e2e.sh + after_script: ./scripts/ci/job_hooks/after_e2e.sh workspaces: create: name: e2e_cache @@ -434,6 +445,7 @@ jobs: - ./scripts/ci/check-env/check-cs-env.sh || travis_terminate 1 - ./scripts/ci/check-env/check-ps-cloud-env.sh || travis_terminate 1 script: ./scripts/travis/e2e/e2e.sh + after_script: ./scripts/ci/job_hooks/after_e2e.sh workspaces: create: name: e2e_cache @@ -456,6 +468,7 @@ jobs: - ./scripts/ci/check-env/check-cs-env.sh || travis_terminate 1 - ./scripts/ci/check-env/check-ps-cloud-env.sh || travis_terminate 1 script: ./scripts/travis/e2e/e2e.sh + after_script: ./scripts/ci/job_hooks/after_e2e.sh workspaces: create: name: e2e_cache @@ -478,6 +491,7 @@ jobs: - ./scripts/ci/check-env/check-cs-env.sh || travis_terminate 1 - ./scripts/ci/check-env/check-ps-cloud-env.sh || travis_terminate 1 script: ./scripts/travis/e2e/e2e.sh + after_script: ./scripts/ci/job_hooks/after_e2e.sh workspaces: create: name: e2e_cache @@ -500,6 +514,7 @@ jobs: - ./scripts/ci/check-env/check-cs-env.sh || travis_terminate 1 - ./scripts/ci/check-env/check-ps-cloud-env.sh || travis_terminate 1 script: ./scripts/travis/e2e/e2e.sh + after_script: ./scripts/ci/job_hooks/after_e2e.sh workspaces: create: name: e2e_cache @@ -522,6 +537,7 @@ jobs: - ./scripts/ci/check-env/check-cs-env.sh || travis_terminate 1 - ./scripts/ci/check-env/check-ps-cloud-env.sh || travis_terminate 1 script: ./scripts/travis/e2e/e2e.sh + after_script: ./scripts/ci/job_hooks/after_e2e.sh workspaces: create: name: e2e_cache diff --git a/scripts/ci/job_hooks/after_e2e.sh b/scripts/ci/job_hooks/after_e2e.sh new file mode 100755 index 00000000000..747ad5332b4 --- /dev/null +++ b/scripts/ci/job_hooks/after_e2e.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +# Upload protractor-smartrunner artifact related to this particular job to S3 +./scripts/ci/utils/artifact-to-s3.sh -a "$SMART_RUNNER_DIRECTORY" -o "$S3_SMART_RUNNER_PATH/$TRAVIS_JOB_ID.tar.bz2" + diff --git a/scripts/ci/job_hooks/before_e2e.sh b/scripts/ci/job_hooks/before_e2e.sh index 44b9ba75d72..50e9128ebbf 100755 --- a/scripts/ci/job_hooks/before_e2e.sh +++ b/scripts/ci/job_hooks/before_e2e.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash + # Download protractor-smartrunner artifact related to this particular job from S3, if exists +./scripts/ci/utils/artifact-from-s3.sh -a "$S3_SMART_RUNNER_PATH/$TRAVIS_JOB_ID.tar.bz2" -o "$SMART_RUNNER_DIRECTORY" + # TODO: This one needs to be cleaned up... Only fixing it like this, because for the current PR it is out of scope # ======================================= rm -rf ./node_modules/@alfresco/adf-cli/ && \ diff --git a/scripts/ci/job_hooks/before_install.sh b/scripts/ci/job_hooks/before_install.sh index 0050f5aebda..e513a183bba 100755 --- a/scripts/ci/job_hooks/before_install.sh +++ b/scripts/ci/job_hooks/before_install.sh @@ -20,6 +20,9 @@ export HEAD_HASH="HEAD" export HEAD_COMMIT_HASH=${TRAVIS_PULL_REQUEST_SHA:-${TRAVIS_COMMIT}} export COMMIT_MESSAGE=`git log --format=%B -n 1 $HEAD_COMMIT_HASH` +# Settings for S3 caching ------------------------------------------------------------- +pip install --user awscli + if [ "${TRAVIS_EVENT_TYPE}" == "push" ]; then echo "push" elif [ "${TRAVIS_EVENT_TYPE}" == "pull_request" ]; then diff --git a/scripts/ci/utils/artifact-from-s3.sh b/scripts/ci/utils/artifact-from-s3.sh new file mode 100755 index 00000000000..9b25ee192f2 --- /dev/null +++ b/scripts/ci/utils/artifact-from-s3.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +show_help() { + echo "Usage: dbpci-artifact-from-s3 " + echo "" + echo "-a or --artifact [mandatory]: path to the s3 artifact (tar.bz2) to download and extract" + echo "-o or --output [mandatory]: directory to extract the archive to" +} + +while [[ $1 == -* ]]; do + case "$1" in + -h|--help|-\?) show_help; exit 0;; + -a|--artifact) ARTIFACT=$2; shift 2;; + -o|--output) OUTPUT=$2; shift 2;; + -*) shift;; + esac +done + +if [ "${ARTIFACT}" == "" ] || [ "${OUTPUT}" == "" ] +then + show_help; + exit 1 +fi + +echo -e "Download from S3 $ARTIFACT to $OUTPUT" +test ! -d $OUTPUT && mkdir -p $OUTPUT + +IS_PRESENT="$(aws s3 ls $ARTIFACT | wc -l | tr -d ' ')" +if [ "${IS_PRESENT}" == "1" ] +then + echo "File ${ARTIFACT} is present. Copying" + aws s3 cp $ARTIFACT ./s3-artifact.tmp + tar -xf ./s3-artifact.tmp -C $OUTPUT + rm ./s3-artifact.tmp +else + echo "File ${ARTIFACT} not present" + exit 1; +fi diff --git a/scripts/ci/utils/artifact-to-s3.sh b/scripts/ci/utils/artifact-to-s3.sh new file mode 100755 index 00000000000..7cf74f98b16 --- /dev/null +++ b/scripts/ci/utils/artifact-to-s3.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +show_help() { + echo "Usage: dbpci-artifact-to-s3 " + echo "" + echo "-a or --artifact [mandatory]: path to the artifact to archieve (tar.bz2) and upload (like ./dist)" + echo "-o or --output [mandatory]: the S3 object to copy it to, like: s3://bucket-name/folder/whatever.tar.bz2" +} + +while [[ $1 == -* ]]; do + case "$1" in + -h|--help|-\?) show_help; exit 0;; + -a|--artifact) ARTIFACT=$2; shift 2;; + -o|--output) OUTPUT=$2; shift 2;; + -*) shift;; + esac +done + +if [ "${ARTIFACT}" == "" ] || [ "${OUTPUT}" == "" ] +then + show_help; + exit 1 +fi + +tar cfj ./s3-artifact.tmp -C $ARTIFACT `ls -A $ARTIFACT` +aws s3 cp ./s3-artifact.tmp $OUTPUT +rm ./s3-artifact.tmp