From e9fa1df0ade64aa9bfc2423e9ac107bb1057bbf1 Mon Sep 17 00:00:00 2001 From: "Alexander V. Tikhonov" Date: Wed, 15 Jul 2020 14:21:49 +0300 Subject: [PATCH] gitlab-ci: set cleanup based on docker for all Set cleanup based on docker for all of the jobs to avoid of fails when the job before change could use docker and reassigned the temporary files in the working directory. If the docker is not available than no need to use it for cleanup and cleanup runs using shell. Gitlab-ci clean flags disabled and reorganized localy as shown at [1]. Disabled gitlab-ci checkout default process as shown at [2] to be able to fix repository before the checkout. Found that previously run gitlab-ci jobs could change permissions files at repository which broke the checkouts at the next jobs. Used for checkout strategy [3], for submodule update strategy [4]. List of steps made localy instead of gitlab-ci preparations: 1. Check/clone the Tarantool repository with submodules. 2. For shell based jobs change ownership of all the sources to 'gitlab-runner' user. (NOTE: in Docker based jobs the 'gitlab-runner' user is not known.) 3. Fetch Tarantool sources with branches and force checkout of the testing commit. 4. Update submodules recursively (use force where supports). 5. Cleanup all the sources from all files except from repository. [1] https://docs.gitlab.com/ee/ci/yaml/README.html#git-clean-flags [2] https://docs.gitlab.com/ee/ci/yaml/README.html#git-strategy [3] https://docs.gitlab.com/ee/ci/yaml/README.html#git-checkout [4] https://docs.gitlab.com/ee/ci/yaml/README.html#git-submodule-strategy Follows up #5036 --- .gitlab-ci.yml | 53 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ba88a26a9c59..ca8d7b2522a7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,18 +4,35 @@ stages: - perf - cleanup +# 1. Git strategy disabled as shown at: +# https://docs.gitlab.com/ee/ci/yaml/README.html#git-strategy +# 2. Git clean flags disabled as shown at: +# https://docs.gitlab.com/ee/ci/yaml/README.html#git-clean-flags variables: GITLAB_MAKE: "make -f .gitlab.mk" - GIT_CLEAN_COMMAND: "git clean -ffdx && git submodule foreach git clean -ffdx && git submodule foreach git status" + GIT_STRATEGY: none GIT_CLEAN_FLAGS: none -.shell_before_script_template: &shell_cleanup_script +# 1. Git checkout strategy used from: +# https://docs.gitlab.com/ee/ci/yaml/README.html#git-checkout +# 2. Submodule strategy used from: +# https://docs.gitlab.com/ee/ci/yaml/README.html#git-submodule-strategy +# List of steps: +# 1) Check/clone the Tarantool repository with submodules. +# 2) For shell based jobs change ownership of all the sources +# to gitlab-runner user (NOTE: in Docker based jobs the +# gitlab-runner user is not known). +# 3) Fetch Tarantool sources with branches and force checkout +# of the testing commit. +# 4) Update submodules recursively (use force where supports). +# 5) Cleanup all the sources from all files except from repository. +.docker_before_script_template: &job_ws_preparation_script before_script: - - /bin/bash -c "${GIT_CLEAN_COMMAND}" - -.docker_before_script_template: &docker_cleanup_script - before_script: - - docker run -w /source -v ${PWD}:/source -i packpack/packpack:el-7 /bin/bash -c "${GIT_CLEAN_COMMAND}" + - /bin/bash -c "pwd && ls -d .git || git clone --recurse-submodules ${CI_REPOSITORY_URL} ." + - /bin/bash -c "! docker -v || (uid=`id -u` ; gid=`id -g` ; docker run -w /source -v ${PWD}:/source -i packpack/packpack:el-7 /bin/bash -c \"chown -R \$uid:\$gid * .[^.]*\")" + - /bin/bash -c "git fetch -p && git checkout -f ${CI_COMMIT_SHORT_SHA}" + - /bin/bash -c "git submodule sync --recursive && git submodule update --force --init --recursive || git submodule update --recursive --init" + - /bin/bash -c "git clean -ffdx && git submodule foreach git clean -ffdx && git submodule foreach git status" # Jobs templates @@ -46,21 +63,21 @@ variables: stage: test tags: - docker_test - <<: *shell_cleanup_script + <<: *job_ws_preparation_script .docker_test_clang8_template: &docker_test_clang8_definition image: "${CI_REGISTRY}/${CI_PROJECT_PATH}/testing/debian-buster:latest" stage: test tags: - docker_test - <<: *shell_cleanup_script + <<: *job_ws_preparation_script .pack_template: &pack_definition <<: *pack_only_definition stage: test tags: - deploy - <<: *docker_cleanup_script + <<: *job_ws_preparation_script script: - ${GITLAB_MAKE} package @@ -69,7 +86,7 @@ variables: stage: test tags: - deploy_test - <<: *docker_cleanup_script + <<: *job_ws_preparation_script script: - ${GITLAB_MAKE} package @@ -78,7 +95,7 @@ variables: stage: test tags: - deploy - <<: *docker_cleanup_script + <<: *job_ws_preparation_script script: - ${GITLAB_MAKE} deploy @@ -87,13 +104,13 @@ variables: stage: test tags: - deploy_test - <<: *docker_cleanup_script + <<: *job_ws_preparation_script script: - ${GITLAB_MAKE} deploy .osx_template: &osx_definition stage: test - <<: *shell_cleanup_script + <<: *job_ws_preparation_script script: - ${GITLAB_MAKE} test_osx @@ -114,14 +131,14 @@ variables: paths: - "*_result.txt" - "*_t_version.txt" - <<: *shell_cleanup_script + <<: *job_ws_preparation_script script: - ${GITLAB_MAKE} perf_run .perf_cleanup_definition: &perf_cleanup_definition <<: *perf_only_definition stage: cleanup - <<: *shell_cleanup_script + <<: *job_ws_preparation_script script: - ${GITLAB_MAKE} perf_cleanup @@ -216,7 +233,7 @@ perf_bootstrap: stage: test tags: - deploy - <<: *shell_cleanup_script + <<: *job_ws_preparation_script script: - ${GITLAB_MAKE} perf_prepare after_script: @@ -534,6 +551,6 @@ static_docker_build: stage: test tags: - deploy_test - <<: *docker_cleanup_script + <<: *job_ws_preparation_script script: - ${GITLAB_MAKE} test_static_docker_build