From fc1e6a3635b8065c9cd71a5ff3a668e1b3594a1b Mon Sep 17 00:00:00 2001 From: Alex Carrie Date: Tue, 23 Mar 2021 10:28:52 +0000 Subject: [PATCH 1/8] APM-1746 Add cache task to build --- azure/common/apigee-build.yml | 16 ++++++++++++++++ scripts/check-python-venv-fixed.sh | 27 +++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 scripts/check-python-venv-fixed.sh diff --git a/azure/common/apigee-build.yml b/azure/common/apigee-build.yml index 68630c2c..1ee8b27e 100644 --- a/azure/common/apigee-build.yml +++ b/azure/common/apigee-build.yml @@ -183,10 +183,26 @@ jobs: # path: "${{ parameters.service_name }}/utils/.venv" # displayName: cache utils dependencies + - task: s3-cache-action@1 + inputs: + key: poetry | $(System.DefaultWorkingDirectory) | ${{ parameters.service_name }}/utils/poetry.lock + location: "${{ parameters.service_name }}/utils/.venv" + debug: true + alias: 'Poetry' + displayName: cache utils dependencies + + - bash: | + cd scripts + export VIRTUAL_ENV_PATH="../.venv" + sh check-python-venv-fixed.sh + exit $? + displayName: check python venv fixed + - bash: | make install sleep 5 workingDirectory: "${{ parameters.service_name }}/utils" + condition: ne(variables['CacheRestored-Poetry'], 'true') displayName: "Install utils " - bash: | diff --git a/scripts/check-python-venv-fixed.sh b/scripts/check-python-venv-fixed.sh new file mode 100644 index 00000000..4df4cbbd --- /dev/null +++ b/scripts/check-python-venv-fixed.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +# this checks the shebang lines are fixed after caching python virtual envs + +set -e + +virtual_env_dir="$VIRTUAL_ENV_PATH" +virtual_env_dir="$(realpath ${virtual_env_dir})" + +echo "${virtual_env_dir}" + +if [ ! -f "${virtual_env_dir}/bin/python" ]; then + echo "bin/python doesn't exist" + exit 0 +fi + +for file in $(find "${virtual_env_dir}/bin" -type f -print0 | xargs -0 file | grep 'Python script' | cut -d: -f1 ); do + echo "Checking file: ${file}" + if (head -n 1 ${file} | grep -Eq '#!.*python'); then + echo "python path fixed for ${file}" + continue + elif (head -n 1 ${file} | grep -Eq "#!${virtual_env_dir}/bin/python"); then + exit 1 + else + echo "no python path to fix for ${file}" + fi +done From 813184c69123171616a3125cc8ca767c641e4117 Mon Sep 17 00:00:00 2001 From: Alex Carrie Date: Tue, 23 Mar 2021 12:30:35 +0000 Subject: [PATCH 2/8] APM-1746 Remove python fix test line --- azure/common/apigee-build.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/azure/common/apigee-build.yml b/azure/common/apigee-build.yml index 1ee8b27e..38ce0fed 100644 --- a/azure/common/apigee-build.yml +++ b/azure/common/apigee-build.yml @@ -191,13 +191,6 @@ jobs: alias: 'Poetry' displayName: cache utils dependencies - - bash: | - cd scripts - export VIRTUAL_ENV_PATH="../.venv" - sh check-python-venv-fixed.sh - exit $? - displayName: check python venv fixed - - bash: | make install sleep 5 From f27833fc0a6b64916f7a70b7756be8fba7e967f4 Mon Sep 17 00:00:00 2001 From: Alex Carrie Date: Tue, 23 Mar 2021 15:30:42 +0000 Subject: [PATCH 3/8] APM-1746 Add pre-install steps --- azure/common/apigee-build.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/azure/common/apigee-build.yml b/azure/common/apigee-build.yml index 38ce0fed..03608c18 100644 --- a/azure/common/apigee-build.yml +++ b/azure/common/apigee-build.yml @@ -24,6 +24,12 @@ parameters: - name: notify type: boolean default: true + - name: pre_install_steps + type: stepList + default: [] + - name: cache_alias + type: string + default: no_cache_alias jobs: - job: build @@ -139,6 +145,9 @@ jobs: inputs: versionSpec: "3.8" + - ${{ each pre_install_step in parameters.pre_install_steps }}: + - ${{ pre_install_step }} + - bash: | n=0 until [ "$n" -ge 3 ] @@ -147,6 +156,7 @@ jobs: n=$((n+1)) done workingDirectory: "${{ parameters.service_name }}" + condition: and( ne(gt(length( ${{ parameters.pre_install_steps }} ), 0), 'true'), ne(variables['CacheRestored-${{ parameters.cache_alias }}'], 'true') ) displayName: Install project dependencies - bash: "make lint" @@ -176,13 +186,6 @@ jobs: - checkout: common path: "s/${{ parameters.service_name }}/utils" -# don't use cache in aws build agents safely, currently -# - task: Cache@2 -# inputs: -# key: poetry | $(Agent.OS) | ${{ parameters.service_name }}/utils/poetry.lock -# path: "${{ parameters.service_name }}/utils/.venv" -# displayName: cache utils dependencies - - task: s3-cache-action@1 inputs: key: poetry | $(System.DefaultWorkingDirectory) | ${{ parameters.service_name }}/utils/poetry.lock From 54dd32fedfb69f5e2a70e348a7eb60d7ba72d5f6 Mon Sep 17 00:00:00 2001 From: Alex Carrie Date: Tue, 23 Mar 2021 16:41:20 +0000 Subject: [PATCH 4/8] APM-1746 Add global_install param --- azure/common/apigee-build.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/azure/common/apigee-build.yml b/azure/common/apigee-build.yml index 03608c18..9b38cd45 100644 --- a/azure/common/apigee-build.yml +++ b/azure/common/apigee-build.yml @@ -24,6 +24,9 @@ parameters: - name: notify type: boolean default: true + - name: global_install + type: boolean + default: true - name: pre_install_steps type: stepList default: [] @@ -156,7 +159,7 @@ jobs: n=$((n+1)) done workingDirectory: "${{ parameters.service_name }}" - condition: and( ne(gt(length( ${{ parameters.pre_install_steps }} ), 0), 'true'), ne(variables['CacheRestored-${{ parameters.cache_alias }}'], 'true') ) + condition: and( eq(${{ parameters.global_install }}, 'true'), ne(variables['CacheRestored-${{ parameters.cache_alias }}'], 'true')) displayName: Install project dependencies - bash: "make lint" From f7ac9dffb9b9fcd410cd306d5173973eb2082f5a Mon Sep 17 00:00:00 2001 From: Alex Carrie Date: Tue, 23 Mar 2021 17:09:24 +0000 Subject: [PATCH 5/8] APM-1746 Check out self at end of build for caching --- azure/common/apigee-build.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/azure/common/apigee-build.yml b/azure/common/apigee-build.yml index 9b38cd45..01b56ede 100644 --- a/azure/common/apigee-build.yml +++ b/azure/common/apigee-build.yml @@ -296,3 +296,8 @@ jobs: state: failure on_failure: true description: "Build failed" + + - checkout: self + path: "s/${{ parameters.service_name }}" + submodules: true + displayName: checkout repo for caching From 1c42e1a1096f747bb850cf43510ffdcf372550ad Mon Sep 17 00:00:00 2001 From: Alex Carrie Date: Wed, 24 Mar 2021 08:58:17 +0000 Subject: [PATCH 6/8] APM-1746 Change cache alias --- azure/common/apigee-build.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/azure/common/apigee-build.yml b/azure/common/apigee-build.yml index 01b56ede..a01715d2 100644 --- a/azure/common/apigee-build.yml +++ b/azure/common/apigee-build.yml @@ -194,14 +194,14 @@ jobs: key: poetry | $(System.DefaultWorkingDirectory) | ${{ parameters.service_name }}/utils/poetry.lock location: "${{ parameters.service_name }}/utils/.venv" debug: true - alias: 'Poetry' + alias: 'Utils' displayName: cache utils dependencies - bash: | make install sleep 5 workingDirectory: "${{ parameters.service_name }}/utils" - condition: ne(variables['CacheRestored-Poetry'], 'true') + condition: ne(variables['CacheRestored-Utils'], 'true') displayName: "Install utils " - bash: | @@ -296,8 +296,3 @@ jobs: state: failure on_failure: true description: "Build failed" - - - checkout: self - path: "s/${{ parameters.service_name }}" - submodules: true - displayName: checkout repo for caching From b5f8d430edbcac0d3865c5c3a7b6b3464d463c74 Mon Sep 17 00:00:00 2001 From: Alex Carrie Date: Wed, 24 Mar 2021 10:03:17 +0000 Subject: [PATCH 7/8] APM-1746 Change to cache steps and strip conditional install --- azure/common/apigee-build.yml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/azure/common/apigee-build.yml b/azure/common/apigee-build.yml index a01715d2..1e8b7d0b 100644 --- a/azure/common/apigee-build.yml +++ b/azure/common/apigee-build.yml @@ -24,15 +24,9 @@ parameters: - name: notify type: boolean default: true - - name: global_install - type: boolean - default: true - - name: pre_install_steps + - name: cache_steps type: stepList default: [] - - name: cache_alias - type: string - default: no_cache_alias jobs: - job: build @@ -148,8 +142,8 @@ jobs: inputs: versionSpec: "3.8" - - ${{ each pre_install_step in parameters.pre_install_steps }}: - - ${{ pre_install_step }} + - ${{ each cache_step in parameters.cache_steps }}: + - ${{ cache_step }} - bash: | n=0 @@ -159,7 +153,6 @@ jobs: n=$((n+1)) done workingDirectory: "${{ parameters.service_name }}" - condition: and( eq(${{ parameters.global_install }}, 'true'), ne(variables['CacheRestored-${{ parameters.cache_alias }}'], 'true')) displayName: Install project dependencies - bash: "make lint" From 5cddbc5faef3adbdd44f085fcde674f2a4b52317 Mon Sep 17 00:00:00 2001 From: Alex Carrie Date: Wed, 24 Mar 2021 10:45:03 +0000 Subject: [PATCH 8/8] APM-1746 Remove check python script --- scripts/check-python-venv-fixed.sh | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 scripts/check-python-venv-fixed.sh diff --git a/scripts/check-python-venv-fixed.sh b/scripts/check-python-venv-fixed.sh deleted file mode 100644 index 4df4cbbd..00000000 --- a/scripts/check-python-venv-fixed.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash - -# this checks the shebang lines are fixed after caching python virtual envs - -set -e - -virtual_env_dir="$VIRTUAL_ENV_PATH" -virtual_env_dir="$(realpath ${virtual_env_dir})" - -echo "${virtual_env_dir}" - -if [ ! -f "${virtual_env_dir}/bin/python" ]; then - echo "bin/python doesn't exist" - exit 0 -fi - -for file in $(find "${virtual_env_dir}/bin" -type f -print0 | xargs -0 file | grep 'Python script' | cut -d: -f1 ); do - echo "Checking file: ${file}" - if (head -n 1 ${file} | grep -Eq '#!.*python'); then - echo "python path fixed for ${file}" - continue - elif (head -n 1 ${file} | grep -Eq "#!${virtual_env_dir}/bin/python"); then - exit 1 - else - echo "no python path to fix for ${file}" - fi -done