Parallelize tests execution

.gitignore

@@ -17,11 +17,11 @@ artifacts
 tmp/gitlab-test
 /.gopath/
 
+testsdefinitions.txt
+/.testoutput/
 /.cover/
+
 codeclimate.json
-coverprofile.html
-coverprofile.txt
-coverprofile.func.txt
 
 # Ignore the generated binary
 /gitlab-runner

.gitlab-ci.yml

@@ -2,6 +2,7 @@ stages:
 - prepare
 - prebuild
 - test
+- coverage
 - build
 - package
 - release
@@ -29,7 +30,7 @@ image: $CI_IMAGE
   - docker:dind
   variables: &docker_variables
     CI_IMAGE: registry.gitlab.com/gitlab-org/gitlab-runner/ci:1.8.7-0
-    DOCKER_DRIVER: overlay
+    DOCKER_DRIVER: overlay2
     DOCKER_HOST: tcp://docker:2375
   tags:
   - docker
@@ -135,6 +136,22 @@ review-docs-cleanup:
   script:
     - ./trigger-build-docs cleanup
 
+tests definitions:
+  stage: prebuild
+  # Using Go 1.10, because the `go test -list` is unavailable in <1.9
+  # and it's much faster in 1.10. And since there is no real test execution
+  # in this step, the version difference doesn't matter
+  image: golang:1.10
+  script:
+  - apt update
+  - apt install make
+  - source ci/touch_make_dependencies
+  - make parallel_test_prepare
+  artifacts:
+    paths:
+    - testsdefinitions.txt
+    expire_in: 7d
+
 #
 # Test stage
 #
@@ -154,37 +171,46 @@ code_quality:
     - ./scripts/codequality analyze -f json --dev | tee gl-code-quality-report.json
   artifacts:
     paths: [gl-code-quality-report.json]
-    expire_in: 1 week
+    expire_in: 7d
 
 .unit_tests: &unit_tests
-  coverage: /total:\s+\(statements\)\s+\d+.\d+\%/
-  artifacts:
-    paths:
-    - coverprofile.html
-    - coverprofile.txt
-    - coverprofile.func.txt
-    expire_in: 1 week
-
-unit tests (no race):
   <<: *docker
-  <<: *unit_tests
   stage: test
-  retry: 2
   script:
+    - JOB_NAME=( $CI_JOB_NAME )
+    - export SUITE_INDEX=${JOB_NAME[-2]}
+    - export SUITE_TOTAL=${JOB_NAME[-1]}
     - source ci/touch_make_dependencies
     - docker pull alpine
     - docker pull docker:dind
     - docker pull docker:git
-    - make test
+    - make parallel_test_execute
+  artifacts:
+    paths:
+      - .cover/*
+      - .testoutput/*
+    when: always
+    expire_in: 7d
 
-unit tests:
-  <<: *docker
+.unit_tests_with_race: &unit_tests_with_race
   <<: *unit_tests
-  stage: test
   allow_failure: true
-  script:
-    - source ci/touch_make_dependencies
-    - CGO_ENABLED=1 TESTFLAGS="-cover -race" make test
+  variables:
+    <<: *docker_variables
+    CGO_ENABLED: "1"
+    TESTFLAGS: "-cover -race"
+
+unit tests 0 5: *unit_tests
+unit tests 1 5: *unit_tests
+unit tests 2 5: *unit_tests
+unit tests 3 5: *unit_tests
+unit tests 4 5: *unit_tests
+
+unit tests with race 0 5: *unit_tests_with_race
+unit tests with race 1 5: *unit_tests_with_race
+unit tests with race 2 5: *unit_tests_with_race
+unit tests with race 3 5: *unit_tests_with_race
+unit tests with race 4 5: *unit_tests_with_race
 
 docs check links:
   image: "registry.gitlab.com/gitlab-org/gitlab-build-images:nanoc-bootstrap-ruby-2.4-alpine"
@@ -224,13 +250,6 @@ integration windows:
   - integration
   - windows
 
-race conditions detector:
-  <<: *docker
-  stage: test
-  script:
-  - source ci/touch_make_dependencies
-  - make check_race_conditions
-
 git 1.7.1:
   <<: *except_docs
   image: centos:6
@@ -249,6 +268,30 @@ git 1.7.1:
   script:
     - make test
 
+#
+# Coverage stage
+#
+
+test coverage report: &test_coverage_report
+  stage: coverage
+  image: golang:1.10
+  coverage: /regular total:\s+\(statements\)\s+\d+.\d+\%/
+  script:
+    - source ci/touch_make_dependencies
+    - make parallel_test_coverage_report
+    - make parallel_test_coverage_report TESTFLAGS="-cover -race"
+  artifacts:
+    paths:
+      - out/coverage/*
+    expire_in: 7d
+
+race conditions detector:
+  stage: coverage
+  image: golang:1.10
+  script:
+  - source ci/touch_make_dependencies
+  - make check_race_conditions
+
 #
 # Build stage
 #

Makefile

@@ -123,7 +123,19 @@ check_race_conditions:
 
 test: $(PKG_BUILD_DIR) docker
 	# Running tests...
-	@./scripts/go_test_with_coverage_report
+	go test $(OUR_PACKAGES) $(TESTFLAGS)
+
+parallel_test_prepare: $(GOPATH_SETUP)
+	# Preparing test commands
+	@./scripts/go_test_with_coverage_report prepare
+
+parallel_test_execute: $(GOPATH_SETUP)
+	# executing tests
+	@./scripts/go_test_with_coverage_report execute
+
+parallel_test_coverage_report: $(GOPATH_SETUP)
+	# Preparing coverage report
+	@./scripts/go_test_with_coverage_report coverage
 
 install:
 	go install --ldflags="$(GO_LDFLAGS)" $(PKG)

scripts/check_race_conditions

@@ -5,15 +5,18 @@
 # fix existing race conditions
 max=33
 
-tmpfile=$(mktemp)
-CGO_ENABLED=1 TESTFLAGS="-race" make test 2>&1 | tee -a "$tmpfile"
+tmpFile=$(mktemp)
 
-cnt=$(grep -c -e "^WARNING: DATA RACE$" "$tmpfile")
+find .testoutput/
+
+grep -E "^WARNING: DATA RACE$" .testoutput/*.race.output.txt > ${tmpFile}
+
+cnt=$(cat ${tmpFile} | wc -l)
 echo "Found ${cnt} race conditions. Maximum allowed value is ${max}"
 
-rm "$tmpfile" 2>/dev/null || true
+rm "$tmpFile" 2>/dev/null || true
 
-if [ "$cnt" -gt "$max" ]; then
+if [ "${cnt}" -gt "${max}" ]; then
   echo "Race conditions count increased"
   exit 1
 fi

scripts/go_test_with_coverage_report

@@ -2,29 +2,152 @@
 
 set -eo pipefail
 
+testsDefinitions="testsdefinitions.txt"
+
+TESTFLAGS=${TESTFLAGS:-"-cover"}
+PARALLEL_TESTS_LIMIT=${PARALLEL_TESTS_LIMIT:-10}
+
+SUITE_TOTAL=${SUITE_TOTAL:-1}
+SUITE_INDEX=${SUITE_INDEX:-0}
+
+output="regular"
 coverMode="count"
 
-if [[ ${TESTFLAGS} = *"-cover"* ]]; then
-    rm -rf ".cover/"
-    mkdir -p ".cover"
+if [[ ${TESTFLAGS} = *"-race"* ]]; then
+    output="race"
+    coverMode="atomic"
+fi
 
-    if [[ ${TESTFLAGS} = *"-race"* ]]; then
-        coverMode="atomic"
-    fi
+printMessage() {
+  echo -e "\033[1m${@}\033[0m"
+}
 
-    echo "Starting go tests with coverprofile in ${coverMode} mode"
+joinBy() {
+    local IFS="${1}"
+    shift
+    echo "${*}"
+}
+
+prepareTestCommands() {
+    [[ ! -f ${testsDefinitions} ]] || rm ${testsDefinitions}
 
     for pkg in ${OUR_PACKAGES}; do
-        profileFile=".cover/$(echo ${pkg} | tr "/" "-").cover"
+        testIndex=0
+        runTests=()
+
+        tests=$(go test -list "Test.*" ${pkg} | grep "^Test" || echo "")
+
+        if [[ -z "${tests}" ]]; then
+            continue
+        fi
+
+        counter=0
+        for test in ${tests}; do
+            counter=$((counter+1))
+            runTests+=("${test}")
+
+            if [[ ${counter} -ge ${PARALLEL_TESTS_LIMIT} ]]; then
+                if [[ ${#runTests[@]} -gt 0 ]]; then
+                    echo ${pkg} ${testIndex} $(joinBy "|" "${runTests[@]}") | tee -a ${testsDefinitions}
+                fi
+
+                counter=0
+                runTests=()
+
+                testIndex=$((testIndex+1))
+            fi
+        done
+
+        if [[ ${#runTests[@]} -gt 0 ]]; then
+            echo ${pkg} ${testIndex} $(joinBy "|" "${runTests[@]}") | tee -a ${testsDefinitions}
+        fi
+    done
+}
+
+executeTestCommand() {
+    local pkg=${1}
+    local index=${2}
+    local runTests=${3}
+
+    local options=""
+
+    local pkgSlug=$(echo ${pkg} | tr "/" "-")
+
+    if [[ ${TESTFLAGS} = *"-cover"* ]]; then
+        mkdir -p ".cover"
+        mkdir -p ".testoutput"
 
-        go test -covermode="${coverMode}" -coverprofile="${profileFile}" ${TESTFLAGS} -v ${pkg}
+        printMessage "\n\n--- Starting part ${index} of go tests of '${pkg}' package with coverprofile in '${coverMode}' mode:\n"
+
+        profileFile=".cover/${pkgSlug}.${index}.${coverMode}.cover.txt"
+        options="-covermode=${coverMode} -coverprofile=${profileFile}"
+    else
+        echo "Starting go test"
+    fi
+
+    testOutputFile=".testoutput/${pkgSlug}.${index}.${output}.output.txt"
+
+    exitCode=0
+    go test ${options} ${TESTFLAGS} -v ${pkg} -run "${runTests}" 2>&1 | tee ${testOutputFile} || exitCode=1
+
+    return $exitCode
+}
+
+executeTestPart() {
+    rm -rf ".cover/"
+    rm -rf ".testoutput/"
+
+    numberOfDefinitions=$(cat ${testsDefinitions} | wc -l)
+    executionSize=$((numberOfDefinitions/SUITE_TOTAL+1))
+    executionOffset=$((SUITE_INDEX*executionSize+1))
+
+    printMessage "Number of definitions: ${numberOfDefinitions}"
+    printMessage "Suite size: ${SUITE_TOTAL}"
+    printMessage "Suite index: ${SUITE_INDEX}"
+
+    printMessage "Execution size: ${executionSize}"
+    printMessage "Execution offset: ${executionOffset}"
+
+    exitCode=0
+    tail -n +${executionOffset} ${testsDefinitions} | head -n ${executionSize} | while read pkg index tests; do
+        executeTestCommand ${pkg} ${index} ${tests} || exitCode=1
     done
 
-    echo "mode: ${coverMode}" > coverprofile.txt
-    grep -h -v "^mode:" .cover/*.cover >> coverprofile.txt
+    exit $exitCode
+}
+
+computeCoverageReport() {
+    local reportDirectory="out/coverage"
+    local sourceFile="${reportDirectory}/coverprofile.${output}.source.txt"
+    local htmlReportFile="${reportDirectory}/coverprofile.${output}.html"
+    local textReportFile="${reportDirectory}/coverprofile.${output}.txt"
+
+    mkdir -p ${reportDirectory}
+
+    echo "mode: ${coverMode}" > ${sourceFile}
+    grep -h -v -e "^mode:" -e "executors/docker/bindata.go" .cover/*.${coverMode}.cover.txt >> ${sourceFile}
+
+    printMessage "Generating HTML coverage report"
+    go tool cover -o ${htmlReportFile} -html=${sourceFile}
+    printMessage "Generating TXT coverage report"
+    go tool cover -o ${textReportFile} -func=${sourceFile}
+
+    printMessage "General coverage percentage:"
+    total=$(grep "total" "${textReportFile}" || echo "")
+
+    if [[ -n "${total}" ]]; then
+        echo "${output} ${total}"
+    fi
+}
 
-    echo "Generating coverprofile.html file"
-    go tool cover -o coverprofile.html -html=coverprofile.txt
-    go tool cover -o coverprofile.func.txt -func=coverprofile.txt
-    grep total coverprofile.func.txt || true
-fi
+case "$1" in
+    prepare)
+        prepareTestCommands
+        ;;
+    execute)
+        executeTestPart
+        ;;
+    coverage)
+        computeCoverageReport
+        ;;
+esac