diff --git a/.gitignore b/.gitignore index 2f3d0c1b0..90c7c730a 100644 --- a/.gitignore +++ b/.gitignore @@ -17,11 +17,11 @@ artifacts tmp/gitlab-test /.gopath/ +testsdefinitions.txt +/.testoutput/ /.cover/ + codeclimate.json -coverprofile.html -coverprofile.txt -coverprofile.func.txt # Ignore the generated binary /gitlab-runner diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9fe98efa5..41e78568c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,6 +2,7 @@ stages: - prepare - prebuild - test +- coverage - build - package - release @@ -29,7 +30,7 @@ image: $CI_IMAGE - docker:dind variables: &docker_variables CI_IMAGE: registry.gitlab.com/gitlab-org/gitlab-runner/ci:1.8.7-0 - DOCKER_DRIVER: overlay + DOCKER_DRIVER: overlay2 DOCKER_HOST: tcp://docker:2375 tags: - docker @@ -135,6 +136,22 @@ review-docs-cleanup: script: - ./trigger-build-docs cleanup +tests definitions: + stage: prebuild + # Using Go 1.10, because the `go test -list` is unavailable in <1.9 + # and it's much faster in 1.10. And since there is no real test execution + # in this step, the version difference doesn't matter + image: golang:1.10 + script: + - apt update + - apt install make + - source ci/touch_make_dependencies + - make parallel_test_prepare + artifacts: + paths: + - testsdefinitions.txt + expire_in: 7d + # # Test stage # @@ -154,37 +171,46 @@ code_quality: - ./scripts/codequality analyze -f json --dev | tee gl-code-quality-report.json artifacts: paths: [gl-code-quality-report.json] - expire_in: 1 week + expire_in: 7d .unit_tests: &unit_tests - coverage: /total:\s+\(statements\)\s+\d+.\d+\%/ - artifacts: - paths: - - coverprofile.html - - coverprofile.txt - - coverprofile.func.txt - expire_in: 1 week - -unit tests (no race): <<: *docker - <<: *unit_tests stage: test - retry: 2 script: + - JOB_NAME=( $CI_JOB_NAME ) + - export SUITE_INDEX=${JOB_NAME[-2]} + - export SUITE_TOTAL=${JOB_NAME[-1]} - source ci/touch_make_dependencies - docker pull alpine - docker pull docker:dind - docker pull docker:git - - make test + - make parallel_test_execute + artifacts: + paths: + - .cover/* + - .testoutput/* + when: always + expire_in: 7d -unit tests: - <<: *docker +.unit_tests_with_race: &unit_tests_with_race <<: *unit_tests - stage: test allow_failure: true - script: - - source ci/touch_make_dependencies - - CGO_ENABLED=1 TESTFLAGS="-cover -race" make test + variables: + <<: *docker_variables + CGO_ENABLED: "1" + TESTFLAGS: "-cover -race" + +unit tests 0 5: *unit_tests +unit tests 1 5: *unit_tests +unit tests 2 5: *unit_tests +unit tests 3 5: *unit_tests +unit tests 4 5: *unit_tests + +unit tests with race 0 5: *unit_tests_with_race +unit tests with race 1 5: *unit_tests_with_race +unit tests with race 2 5: *unit_tests_with_race +unit tests with race 3 5: *unit_tests_with_race +unit tests with race 4 5: *unit_tests_with_race docs check links: image: "registry.gitlab.com/gitlab-org/gitlab-build-images:nanoc-bootstrap-ruby-2.4-alpine" @@ -224,13 +250,6 @@ integration windows: - integration - windows -race conditions detector: - <<: *docker - stage: test - script: - - source ci/touch_make_dependencies - - make check_race_conditions - git 1.7.1: <<: *except_docs image: centos:6 @@ -249,6 +268,30 @@ git 1.7.1: script: - make test +# +# Coverage stage +# + +test coverage report: &test_coverage_report + stage: coverage + image: golang:1.10 + coverage: /regular total:\s+\(statements\)\s+\d+.\d+\%/ + script: + - source ci/touch_make_dependencies + - make parallel_test_coverage_report + - make parallel_test_coverage_report TESTFLAGS="-cover -race" + artifacts: + paths: + - out/coverage/* + expire_in: 7d + +race conditions detector: + stage: coverage + image: golang:1.10 + script: + - source ci/touch_make_dependencies + - make check_race_conditions + # # Build stage # diff --git a/Makefile b/Makefile index 90bd1e450..87b083396 100644 --- a/Makefile +++ b/Makefile @@ -123,7 +123,19 @@ check_race_conditions: test: $(PKG_BUILD_DIR) docker # Running tests... - @./scripts/go_test_with_coverage_report + go test $(OUR_PACKAGES) $(TESTFLAGS) + +parallel_test_prepare: $(GOPATH_SETUP) + # Preparing test commands + @./scripts/go_test_with_coverage_report prepare + +parallel_test_execute: $(GOPATH_SETUP) + # executing tests + @./scripts/go_test_with_coverage_report execute + +parallel_test_coverage_report: $(GOPATH_SETUP) + # Preparing coverage report + @./scripts/go_test_with_coverage_report coverage install: go install --ldflags="$(GO_LDFLAGS)" $(PKG) diff --git a/scripts/check_race_conditions b/scripts/check_race_conditions index ddca02bcb..9ddecdc13 100755 --- a/scripts/check_race_conditions +++ b/scripts/check_race_conditions @@ -5,15 +5,18 @@ # fix existing race conditions max=33 -tmpfile=$(mktemp) -CGO_ENABLED=1 TESTFLAGS="-race" make test 2>&1 | tee -a "$tmpfile" +tmpFile=$(mktemp) -cnt=$(grep -c -e "^WARNING: DATA RACE$" "$tmpfile") +find .testoutput/ + +grep -E "^WARNING: DATA RACE$" .testoutput/*.race.output.txt > ${tmpFile} + +cnt=$(cat ${tmpFile} | wc -l) echo "Found ${cnt} race conditions. Maximum allowed value is ${max}" -rm "$tmpfile" 2>/dev/null || true +rm "$tmpFile" 2>/dev/null || true -if [ "$cnt" -gt "$max" ]; then +if [ "${cnt}" -gt "${max}" ]; then echo "Race conditions count increased" exit 1 fi diff --git a/scripts/go_test_with_coverage_report b/scripts/go_test_with_coverage_report index dc2ded403..9f21d7635 100755 --- a/scripts/go_test_with_coverage_report +++ b/scripts/go_test_with_coverage_report @@ -2,29 +2,152 @@ set -eo pipefail +testsDefinitions="testsdefinitions.txt" + +TESTFLAGS=${TESTFLAGS:-"-cover"} +PARALLEL_TESTS_LIMIT=${PARALLEL_TESTS_LIMIT:-10} + +SUITE_TOTAL=${SUITE_TOTAL:-1} +SUITE_INDEX=${SUITE_INDEX:-0} + +output="regular" coverMode="count" -if [[ ${TESTFLAGS} = *"-cover"* ]]; then - rm -rf ".cover/" - mkdir -p ".cover" +if [[ ${TESTFLAGS} = *"-race"* ]]; then + output="race" + coverMode="atomic" +fi - if [[ ${TESTFLAGS} = *"-race"* ]]; then - coverMode="atomic" - fi +printMessage() { + echo -e "\033[1m${@}\033[0m" +} - echo "Starting go tests with coverprofile in ${coverMode} mode" +joinBy() { + local IFS="${1}" + shift + echo "${*}" +} + +prepareTestCommands() { + [[ ! -f ${testsDefinitions} ]] || rm ${testsDefinitions} for pkg in ${OUR_PACKAGES}; do - profileFile=".cover/$(echo ${pkg} | tr "/" "-").cover" + testIndex=0 + runTests=() + + tests=$(go test -list "Test.*" ${pkg} | grep "^Test" || echo "") + + if [[ -z "${tests}" ]]; then + continue + fi + + counter=0 + for test in ${tests}; do + counter=$((counter+1)) + runTests+=("${test}") + + if [[ ${counter} -ge ${PARALLEL_TESTS_LIMIT} ]]; then + if [[ ${#runTests[@]} -gt 0 ]]; then + echo ${pkg} ${testIndex} $(joinBy "|" "${runTests[@]}") | tee -a ${testsDefinitions} + fi + + counter=0 + runTests=() + + testIndex=$((testIndex+1)) + fi + done + + if [[ ${#runTests[@]} -gt 0 ]]; then + echo ${pkg} ${testIndex} $(joinBy "|" "${runTests[@]}") | tee -a ${testsDefinitions} + fi + done +} + +executeTestCommand() { + local pkg=${1} + local index=${2} + local runTests=${3} + + local options="" + + local pkgSlug=$(echo ${pkg} | tr "/" "-") + + if [[ ${TESTFLAGS} = *"-cover"* ]]; then + mkdir -p ".cover" + mkdir -p ".testoutput" - go test -covermode="${coverMode}" -coverprofile="${profileFile}" ${TESTFLAGS} -v ${pkg} + printMessage "\n\n--- Starting part ${index} of go tests of '${pkg}' package with coverprofile in '${coverMode}' mode:\n" + + profileFile=".cover/${pkgSlug}.${index}.${coverMode}.cover.txt" + options="-covermode=${coverMode} -coverprofile=${profileFile}" + else + echo "Starting go test" + fi + + testOutputFile=".testoutput/${pkgSlug}.${index}.${output}.output.txt" + + exitCode=0 + go test ${options} ${TESTFLAGS} -v ${pkg} -run "${runTests}" 2>&1 | tee ${testOutputFile} || exitCode=1 + + return $exitCode +} + +executeTestPart() { + rm -rf ".cover/" + rm -rf ".testoutput/" + + numberOfDefinitions=$(cat ${testsDefinitions} | wc -l) + executionSize=$((numberOfDefinitions/SUITE_TOTAL+1)) + executionOffset=$((SUITE_INDEX*executionSize+1)) + + printMessage "Number of definitions: ${numberOfDefinitions}" + printMessage "Suite size: ${SUITE_TOTAL}" + printMessage "Suite index: ${SUITE_INDEX}" + + printMessage "Execution size: ${executionSize}" + printMessage "Execution offset: ${executionOffset}" + + exitCode=0 + tail -n +${executionOffset} ${testsDefinitions} | head -n ${executionSize} | while read pkg index tests; do + executeTestCommand ${pkg} ${index} ${tests} || exitCode=1 done - echo "mode: ${coverMode}" > coverprofile.txt - grep -h -v "^mode:" .cover/*.cover >> coverprofile.txt + exit $exitCode +} + +computeCoverageReport() { + local reportDirectory="out/coverage" + local sourceFile="${reportDirectory}/coverprofile.${output}.source.txt" + local htmlReportFile="${reportDirectory}/coverprofile.${output}.html" + local textReportFile="${reportDirectory}/coverprofile.${output}.txt" + + mkdir -p ${reportDirectory} + + echo "mode: ${coverMode}" > ${sourceFile} + grep -h -v -e "^mode:" -e "executors/docker/bindata.go" .cover/*.${coverMode}.cover.txt >> ${sourceFile} + + printMessage "Generating HTML coverage report" + go tool cover -o ${htmlReportFile} -html=${sourceFile} + printMessage "Generating TXT coverage report" + go tool cover -o ${textReportFile} -func=${sourceFile} + + printMessage "General coverage percentage:" + total=$(grep "total" "${textReportFile}" || echo "") + + if [[ -n "${total}" ]]; then + echo "${output} ${total}" + fi +} - echo "Generating coverprofile.html file" - go tool cover -o coverprofile.html -html=coverprofile.txt - go tool cover -o coverprofile.func.txt -func=coverprofile.txt - grep total coverprofile.func.txt || true -fi \ No newline at end of file +case "$1" in + prepare) + prepareTestCommands + ;; + execute) + executeTestPart + ;; + coverage) + computeCoverageReport + ;; +esac \ No newline at end of file