From fca966a28de22b70080004440448e8f8dbb86d68 Mon Sep 17 00:00:00 2001 From: "Alexander V. Tikhonov" Date: Wed, 15 Jul 2020 14:21:49 +0300 Subject: [PATCH] gitlab-ci: setup local cleanup/checkout processes Set cleanup based on docker for all of the jobs to avoid of fails when the job before change could use docker and reassigned the temporary files in the working directory. If the docker is not available than no need to use it for cleanup and cleanup runs using shell. Gitlab-ci clean flags disabled and reorganized localy as shown at [1]. Disabled gitlab-ci checkout default process as shown at [2] to be able to fix repository before the checkout. Found that previously run gitlab-ci jobs could change permissions files at repository which broke the checkouts at the next jobs. Used for checkout strategy [3], for submodule update strategy [4]. Submodules local update routine in .gitlab.mk file became unneeded and removed. List of steps made localy instead of gitlab-ci preparations: 1. Check/clone the Tarantool repository with submodules. 2. For shell based jobs change ownership of all the sources to 'gitlab-runner' user. (NOTE: in Docker based jobs the 'gitlab-runner' user is not known.) 3. Fetch Tarantool sources with branches and force checkout of the testing commit. 4. Update submodules recursively (use force where supports). 5. Cleanup all the sources from all files except from repository. [1] https://docs.gitlab.com/ee/ci/yaml/README.html#git-clean-flags [2] https://docs.gitlab.com/ee/ci/yaml/README.html#git-strategy [3] https://docs.gitlab.com/ee/ci/yaml/README.html#git-checkout [4] https://docs.gitlab.com/ee/ci/yaml/README.html#git-submodule-strategy Follows up #5036 --- .gitlab-ci.yml | 47 +++++++++++++++++++++++++---------------------- .gitlab.mk | 17 ++--------------- 2 files changed, 27 insertions(+), 37 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ba88a26a9c59..580825005d40 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,18 +4,34 @@ stages: - perf - cleanup +# 1. Git strategy disabled as shown at: +# https://docs.gitlab.com/ee/ci/yaml/README.html#git-strategy +# 2. Git clean flags disabled as shown at: +# https://docs.gitlab.com/ee/ci/yaml/README.html#git-clean-flags variables: GITLAB_MAKE: "make -f .gitlab.mk" - GIT_CLEAN_COMMAND: "git clean -ffdx && git submodule foreach git clean -ffdx && git submodule foreach git status" + GIT_STRATEGY: none GIT_CLEAN_FLAGS: none -.shell_before_script_template: &shell_cleanup_script - before_script: - - /bin/bash -c "${GIT_CLEAN_COMMAND}" - -.docker_before_script_template: &docker_cleanup_script - before_script: - - docker run -w /source -v ${PWD}:/source -i packpack/packpack:el-7 /bin/bash -c "${GIT_CLEAN_COMMAND}" +# 1. Git checkout strategy used from: +# https://docs.gitlab.com/ee/ci/yaml/README.html#git-checkout +# 2. Submodule strategy used from: +# https://docs.gitlab.com/ee/ci/yaml/README.html#git-submodule-strategy +# List of steps: +# 1) Check/clone the Tarantool repository with submodules. +# 2) For shell based jobs change ownership of all the sources +# to gitlab-runner user (NOTE: in Docker based jobs the +# gitlab-runner user is not known). +# 3) Fetch Tarantool sources with branches and force checkout +# of the testing commit. +# 4) Update submodules recursively (use force where supports). +# 5) Cleanup all the sources from all files except from repository. +before_script: + - /bin/bash -c "pwd && ls -d .git || git clone --recurse-submodules ${CI_REPOSITORY_URL} ." + - /bin/bash -c "! docker -v || (uid=`id -u` ; gid=`id -g` ; docker run -w /source -v ${PWD}:/source -i packpack/packpack:el-7 /bin/bash -c \"chown -R \$uid:\$gid * .[^.]*\")" + - /bin/bash -c "git fetch -p && git checkout -f ${CI_COMMIT_SHORT_SHA}" + - /bin/bash -c "git submodule sync --recursive && git submodule update --force --init --recursive || git submodule update --recursive --init" + - /bin/bash -c "git clean -ffdx && git submodule foreach git clean -ffdx && git submodule foreach git status" # Jobs templates @@ -46,21 +62,18 @@ variables: stage: test tags: - docker_test - <<: *shell_cleanup_script .docker_test_clang8_template: &docker_test_clang8_definition image: "${CI_REGISTRY}/${CI_PROJECT_PATH}/testing/debian-buster:latest" stage: test tags: - docker_test - <<: *shell_cleanup_script .pack_template: &pack_definition <<: *pack_only_definition stage: test tags: - deploy - <<: *docker_cleanup_script script: - ${GITLAB_MAKE} package @@ -69,7 +82,6 @@ variables: stage: test tags: - deploy_test - <<: *docker_cleanup_script script: - ${GITLAB_MAKE} package @@ -78,7 +90,6 @@ variables: stage: test tags: - deploy - <<: *docker_cleanup_script script: - ${GITLAB_MAKE} deploy @@ -87,21 +98,16 @@ variables: stage: test tags: - deploy_test - <<: *docker_cleanup_script script: - ${GITLAB_MAKE} deploy .osx_template: &osx_definition stage: test - <<: *shell_cleanup_script script: - ${GITLAB_MAKE} test_osx .vbox_template: &vbox_definition stage: test - before_script: - - /bin/bash -c "${GIT_CLEAN_COMMAND}" - - ${GITLAB_MAKE} vms_start after_script: - ${GITLAB_MAKE} vms_shutdown @@ -114,14 +120,12 @@ variables: paths: - "*_result.txt" - "*_t_version.txt" - <<: *shell_cleanup_script script: - ${GITLAB_MAKE} perf_run .perf_cleanup_definition: &perf_cleanup_definition <<: *perf_only_definition stage: cleanup - <<: *shell_cleanup_script script: - ${GITLAB_MAKE} perf_cleanup @@ -203,6 +207,7 @@ freebsd_12_release: VMS_PORT: '2232' MAKE: 'gmake' script: + - ${GITLAB_MAKE} vms_start - ${GITLAB_MAKE} vms_test_freebsd # #### @@ -216,7 +221,6 @@ perf_bootstrap: stage: test tags: - deploy - <<: *shell_cleanup_script script: - ${GITLAB_MAKE} perf_prepare after_script: @@ -534,6 +538,5 @@ static_docker_build: stage: test tags: - deploy_test - <<: *docker_cleanup_script script: - ${GITLAB_MAKE} test_static_docker_build diff --git a/.gitlab.mk b/.gitlab.mk index f2a9b77fde4c..0ce978063e6f 100644 --- a/.gitlab.mk +++ b/.gitlab.mk @@ -1,20 +1,8 @@ GITLAB_MAKE:=${MAKE} -f .gitlab.mk TRAVIS_MAKE:=${MAKE} -f .travis.mk -# ##### -# Utils -# ##### - -# Update submodules. -# -# Note: There is no --force option for `git submodule` on git -# 1.7.1, which is shiped in CentOS 6. -git_submodule_update: - git submodule update --force --recursive --init 2>/dev/null || \ - git submodule update --recursive --init - # Pass *_no_deps goals to .travis.mk. -test_%: git_submodule_update +test_%: ${TRAVIS_MAKE} $@ # ####################################################### @@ -107,7 +95,6 @@ vms_test_%: ssh ${VMS_USER}@127.0.0.1 -p ${VMS_PORT} "/bin/bash -c \ '${EXTRA_ENV} \ cd tarantool && \ - ${GITLAB_MAKE} git_submodule_update && \ ${TRAVIS_MAKE} $(subst vms_,,$@)'" vms_shutdown: @@ -122,7 +109,7 @@ MAJOR_VERSION=$(word 1,$(subst ., ,$(GIT_DESCRIBE))) MINOR_VERSION=$(word 2,$(subst ., ,$(GIT_DESCRIBE))) BUCKET="$(MAJOR_VERSION).$(MINOR_VERSION)" -deploy_prepare: git_submodule_update +deploy_prepare: [ -d packpack ] || \ git clone https://github.com/packpack/packpack.git packpack rm -rf build