diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index c35200defb9357..3fa88b78b6db04 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -119,10 +119,6 @@ jobs:
- name: test
shell: bash
run: . /etc/profile && ci/run-test-slice.sh ${{matrix.nr}} 10
- - name: ci/print-test-failures.sh
- if: failure()
- shell: bash
- run: ci/print-test-failures.sh
- name: Upload failed tests' directories
if: failure() && env.FAILED_TEST_ARTIFACTS != ''
uses: actions/upload-artifact@v2
@@ -204,10 +200,6 @@ jobs:
env:
NO_SVN_TESTS: 1
run: . /etc/profile && ci/run-test-slice.sh ${{matrix.nr}} 10
- - name: ci/print-test-failures.sh
- if: failure()
- shell: bash
- run: ci/print-test-failures.sh
- name: Upload failed tests' directories
if: failure() && env.FAILED_TEST_ARTIFACTS != ''
uses: actions/upload-artifact@v2
@@ -261,8 +253,6 @@ jobs:
- uses: actions/checkout@v2
- run: ci/install-dependencies.sh
- run: ci/run-build-and-tests.sh
- - run: ci/print-test-failures.sh
- if: failure()
- name: Upload failed tests' directories
if: failure() && env.FAILED_TEST_ARTIFACTS != ''
uses: actions/upload-artifact@v2
@@ -292,8 +282,6 @@ jobs:
- uses: actions/checkout@v1
- run: ci/install-docker-dependencies.sh
- run: ci/run-build-and-tests.sh
- - run: ci/print-test-failures.sh
- if: failure()
- name: Upload failed tests' directories
if: failure() && env.FAILED_TEST_ARTIFACTS != ''
uses: actions/upload-artifact@v1
diff --git a/ci/lib.sh b/ci/lib.sh
index cbc2f8f1caa6c0..28b1ba2dc545cc 100755
--- a/ci/lib.sh
+++ b/ci/lib.sh
@@ -1,5 +1,49 @@
# Library of functions shared by all CI scripts
+if test true != "$GITHUB_ACTIONS"
+then
+ begin_group () { :; }
+ end_group () { :; }
+
+ group () {
+ shift
+ "$@"
+ }
+ set -x
+else
+ begin_group () {
+ need_to_end_group=t
+ echo "::group::$1" >&2
+ set -x
+ }
+
+ end_group () {
+ test -n "$need_to_end_group" || return 0
+ set +x
+ need_to_end_group=
+ echo '::endgroup::' >&2
+ }
+ trap end_group EXIT
+
+ group () {
+ set +x
+ begin_group "$1"
+ shift
+ "$@"
+ res=$?
+ end_group
+ return $res
+ }
+
+ begin_group "CI setup"
+fi
+
+# Set 'exit on error' for all CI scripts to let the caller know that
+# something went wrong.
+# Set tracing executed commands, primarily setting environment variables
+# and installing dependencies.
+set -e
+
skip_branch_tip_with_tag () {
# Sometimes, a branch is pushed at the same time the tag that points
# at the same commit as the tip of the branch is pushed, and building
@@ -69,8 +113,7 @@ skip_good_tree () {
exit 0
}
-check_unignored_build_artifacts ()
-{
+check_unignored_build_artifacts () {
! git ls-files --other --exclude-standard --error-unmatch \
-- ':/*' 2>/dev/null ||
{
@@ -79,18 +122,16 @@ check_unignored_build_artifacts ()
}
}
+handle_failed_tests () {
+ return 1
+}
+
# GitHub Action doesn't set TERM, which is required by tput
export TERM=${TERM:-dumb}
# Clear MAKEFLAGS that may come from the outside world.
export MAKEFLAGS=
-# Set 'exit on error' for all CI scripts to let the caller know that
-# something went wrong.
-# Set tracing executed commands, primarily setting environment variables
-# and installing dependencies.
-set -ex
-
if test -n "$SYSTEM_COLLECTIONURI" || test -n "$SYSTEM_TASKDEFINITIONSURI"
then
CI_TYPE=azure-pipelines
@@ -124,11 +165,30 @@ then
CI_JOB_ID="$GITHUB_RUN_ID"
CC="${CC:-gcc}"
DONT_SKIP_TAGS=t
+ handle_failed_tests () {
+ mkdir -p t/failed-test-artifacts
+ echo "FAILED_TEST_ARTIFACTS=t/failed-test-artifacts" >>$GITHUB_ENV
+
+ for test_exit in t/test-results/*.exit
+ do
+ test 0 != "$(cat "$test_exit")" || continue
+
+ test_name="${test_exit%.exit}"
+ test_name="${test_name##*/}"
+ printf "\\e[33m\\e[1m=== Failed test: ${test_name} ===\\e[m\\n"
+ cat "t/test-results/$test_name.markup"
+
+ trash_dir="t/trash directory.$test_name"
+ cp "t/test-results/$test_name.out" t/failed-test-artifacts/
+ tar czf t/failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir"
+ done
+ return 1
+ }
cache_dir="$HOME/none"
export GIT_PROVE_OPTS="--timer --jobs 10"
- export GIT_TEST_OPTS="--verbose-log -x"
+ export GIT_TEST_OPTS="--verbose-log -x --github-workflow-markup"
MAKEFLAGS="$MAKEFLAGS --jobs=10"
test windows != "$CI_OS_NAME" ||
GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS"
@@ -211,3 +271,6 @@ linux-leaks)
esac
MAKEFLAGS="$MAKEFLAGS CC=${CC:-cc}"
+
+end_group
+set -x
diff --git a/ci/run-build-and-tests.sh b/ci/run-build-and-tests.sh
index d671f402784a40..88806db5911dd6 100755
--- a/ci/run-build-and-tests.sh
+++ b/ci/run-build-and-tests.sh
@@ -10,7 +10,7 @@ windows*) cmd //c mklink //j t\\.prove "$(cygpath -aw "$cache_dir/.prove")";;
*) ln -s "$cache_dir/.prove" t/.prove;;
esac
-export MAKE_TARGETS="all test"
+run_tests=t
case "$jobname" in
linux-gcc)
@@ -41,14 +41,19 @@ pedantic)
# Don't run the tests; we only care about whether Git can be
# built.
export DEVOPTS=pedantic
- export MAKE_TARGETS=all
+ run_tests=
;;
esac
# Any new "test" targets should not go after this "make", but should
# adjust $MAKE_TARGETS. Otherwise compilation-only targets above will
# start running tests.
-make $MAKE_TARGETS
+group Build make
+if test -n "$run_tests"
+then
+ group "Run tests" make test ||
+ handle_failed_tests
+fi
check_unignored_build_artifacts
save_good_tree
diff --git a/ci/run-test-slice.sh b/ci/run-test-slice.sh
index f8c2c3106a2ef4..a3c67956a8df8f 100755
--- a/ci/run-test-slice.sh
+++ b/ci/run-test-slice.sh
@@ -10,8 +10,9 @@ windows*) cmd //c mklink //j t\\.prove "$(cygpath -aw "$cache_dir/.prove")";;
*) ln -s "$cache_dir/.prove" t/.prove;;
esac
-make --quiet -C t T="$(cd t &&
+group "Run tests" make --quiet -C t T="$(cd t &&
./helper/test-tool path-utils slice-tests "$1" "$2" t[0-9]*.sh |
- tr '\n' ' ')"
+ tr '\n' ' ')" ||
+handle_failed_tests
check_unignored_build_artifacts
diff --git a/t/test-lib-functions.sh b/t/test-lib-functions.sh
index 0f439c99d61091..4e86f7d9d0e439 100644
--- a/t/test-lib-functions.sh
+++ b/t/test-lib-functions.sh
@@ -719,7 +719,7 @@ test_verify_prereq () {
}
test_expect_failure () {
- test_start_
+ test_start_ "$@"
test "$#" = 3 && { test_prereq=$1; shift; } || test_prereq=
test "$#" = 2 ||
BUG "not 2 or 3 parameters to test-expect-failure"
@@ -739,7 +739,7 @@ test_expect_failure () {
}
test_expect_success () {
- test_start_
+ test_start_ "$@"
test "$#" = 3 && { test_prereq=$1; shift; } || test_prereq=
test "$#" = 2 ||
BUG "not 2 or 3 parameters to test-expect-success"
diff --git a/t/test-lib-github-workflow-markup.sh b/t/test-lib-github-workflow-markup.sh
new file mode 100644
index 00000000000000..d8dc969df4a17b
--- /dev/null
+++ b/t/test-lib-github-workflow-markup.sh
@@ -0,0 +1,50 @@
+# Library of functions to mark up test scripts' output suitable for
+# pretty-printing it in GitHub workflows.
+#
+# Copyright (c) 2022 Johannes Schindelin
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see http://www.gnu.org/licenses/ .
+#
+# The idea is for `test-lib.sh` to source this file when run in GitHub
+# workflows; these functions will then override (empty) functions
+# that are are called at the appropriate times during the test runs.
+
+start_test_output () {
+ test -n "$GIT_TEST_TEE_OUTPUT_FILE" ||
+ die "--github-workflow-markup requires --verbose-log"
+ github_markup_output="${GIT_TEST_TEE_OUTPUT_FILE%.out}.markup"
+ >$github_markup_output
+ GIT_TEST_TEE_OFFSET=0
+}
+
+# No need to override start_test_case_output
+
+finalize_test_case_output () {
+ test_case_result=$1
+ shift
+ case "$test_case_result" in
+ failure)
+ echo >>$github_markup_output "::error::failed: $this_test.$test_count $1"
+ ;;
+ fixed)
+ echo >>$github_markup_output "::notice::fixed: $this_test.$test_count $1"
+ ;;
+ esac
+ echo >>$github_markup_output "::group::$test_case_result: $this_test.$test_count $*"
+ test-tool >>$github_markup_output path-utils skip-n-bytes \
+ "$GIT_TEST_TEE_OUTPUT_FILE" $GIT_TEST_TEE_OFFSET
+ echo >>$github_markup_output "::endgroup::"
+}
+
+# No need to override finalize_test_output
diff --git a/t/test-lib-junit.sh b/t/test-lib-junit.sh
new file mode 100644
index 00000000000000..c959183c7e2c84
--- /dev/null
+++ b/t/test-lib-junit.sh
@@ -0,0 +1,132 @@
+# Library of functions to format test scripts' output in JUnit XML
+# format, to support Git's test suite result to be presented in an
+# easily digestible way on Azure Pipelines.
+#
+# Copyright (c) 2022 Johannes Schindelin
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see http://www.gnu.org/licenses/ .
+#
+# The idea is for `test-lib.sh` to source this file when the user asks
+# for JUnit XML; these functions will then override (empty) functions
+# that are are called at the appropriate times during the test runs.
+
+start_test_output () {
+ junit_xml_dir="$TEST_OUTPUT_DIRECTORY/out"
+ mkdir -p "$junit_xml_dir"
+ junit_xml_base=${1##*/}
+ junit_xml_path="$junit_xml_dir/TEST-${junit_xml_base%.sh}.xml"
+ junit_attrs="name=\"${junit_xml_base%.sh}\""
+ junit_attrs="$junit_attrs timestamp=\"$(TZ=UTC \
+ date +%Y-%m-%dT%H:%M:%S)\""
+ write_junit_xml --truncate "" " "
+ junit_suite_start=$(test-tool date getnanos)
+ if test -n "$GIT_TEST_TEE_OUTPUT_FILE"
+ then
+ GIT_TEST_TEE_OFFSET=0
+ fi
+}
+
+start_test_case_output () {
+ junit_start=$(test-tool date getnanos)
+}
+
+finalize_test_case_output () {
+ test_case_result=$1
+ shift
+ case "$test_case_result" in
+ ok)
+ set "$*"
+ ;;
+ failure)
+ junit_insert=""
+ junit_insert="$junit_insert $(xml_attr_encode \
+ "$(if test -n "$GIT_TEST_TEE_OUTPUT_FILE"
+ then
+ test-tool path-utils skip-n-bytes \
+ "$GIT_TEST_TEE_OUTPUT_FILE" $GIT_TEST_TEE_OFFSET
+ else
+ printf '%s\n' "$@" | sed 1d
+ fi)")"
+ junit_insert="$junit_insert"
+ if test -n "$GIT_TEST_TEE_OUTPUT_FILE"
+ then
+ junit_insert="$junit_insert$(xml_attr_encode \
+ "$(cat "$GIT_TEST_TEE_OUTPUT_FILE")")"
+ fi
+ set "$1" " $junit_insert"
+ ;;
+ fixed)
+ set "$* (breakage fixed)"
+ ;;
+ broken)
+ set "$* (known breakage)"
+ ;;
+ skip)
+ message="$(xml_attr_encode --no-lf "$skipped_reason")"
+ set "$1" " "
+ ;;
+ esac
+
+ junit_attrs="name=\"$(xml_attr_encode --no-lf "$this_test.$test_count $1")\""
+ shift
+ junit_attrs="$junit_attrs classname=\"$this_test\""
+ junit_attrs="$junit_attrs time=\"$(test-tool \
+ date getnanos $junit_start)\""
+ write_junit_xml "$(printf '%s\n' \
+ " " "$@" " ")"
+ junit_have_testcase=t
+}
+
+finalize_test_output () {
+ if test -n "$junit_xml_path"
+ then
+ test -n "$junit_have_testcase" || {
+ junit_start=$(test-tool date getnanos)
+ write_junit_xml_testcase "all tests skipped"
+ }
+
+ # adjust the overall time
+ junit_time=$(test-tool date getnanos $junit_suite_start)
+ sed -e "s/\(]*/& time=\"$junit_time\"/" \
+ -e '/^ *<\/testsuite/d' \
+ <"$junit_xml_path" >"$junit_xml_path.new"
+ mv "$junit_xml_path.new" "$junit_xml_path"
+
+ write_junit_xml " " ""
+ write_junit_xml=
+ fi
+}
+
+write_junit_xml () {
+ case "$1" in
+ --truncate)
+ >"$junit_xml_path"
+ junit_have_testcase=
+ shift
+ ;;
+ esac
+ printf '%s\n' "$@" >>"$junit_xml_path"
+}
+
+xml_attr_encode () {
+ if test "x$1" = "x--no-lf"
+ then
+ shift
+ printf '%s' "$*" | test-tool xml-encode
+ else
+ printf '%s\n' "$@" | test-tool xml-encode
+ fi
+}
diff --git a/t/test-lib.sh b/t/test-lib.sh
index 06da15a8b9c0f7..30c4b1e59f8146 100644
--- a/t/test-lib.sh
+++ b/t/test-lib.sh
@@ -137,6 +137,12 @@ mark_option_requires_arg () {
store_arg_to=$2
}
+# These functions can be overridden e.g. to output JUnit XML
+start_test_output () { :; }
+start_test_case_output () { :; }
+finalize_test_case_output () { :; }
+finalize_test_output () { :; }
+
parse_option () {
local opt="$1"
@@ -196,7 +202,10 @@ parse_option () {
tee=t
;;
--write-junit-xml)
- write_junit_xml=t
+ . "$TEST_DIRECTORY/test-lib-junit.sh"
+ ;;
+ --github-workflow-markup)
+ . "$TEST_DIRECTORY/test-lib-github-workflow-markup.sh"
;;
--stress)
stress=t ;;
@@ -645,7 +654,7 @@ exec 6<&0
exec 7>&2
_error_exit () {
- finalize_junit_xml
+ finalize_test_output
GIT_EXIT_OK=t
exit 1
}
@@ -755,58 +764,31 @@ trap '{ code=$?; set +x; } 2>/dev/null; exit $code' INT TERM HUP
# the test_expect_* functions instead.
test_ok_ () {
- if test -n "$write_junit_xml"
- then
- write_junit_xml_testcase "$*"
- fi
test_success=$(($test_success + 1))
say_color "" "ok $test_count - $@"
+ finalize_test_case_output ok "$@"
}
test_failure_ () {
- if test -n "$write_junit_xml"
- then
- junit_insert=""
- junit_insert="$junit_insert $(xml_attr_encode \
- "$(if test -n "$GIT_TEST_TEE_OUTPUT_FILE"
- then
- test-tool path-utils skip-n-bytes \
- "$GIT_TEST_TEE_OUTPUT_FILE" $GIT_TEST_TEE_OFFSET
- else
- printf '%s\n' "$@" | sed 1d
- fi)")"
- junit_insert="$junit_insert"
- if test -n "$GIT_TEST_TEE_OUTPUT_FILE"
- then
- junit_insert="$junit_insert$(xml_attr_encode \
- "$(cat "$GIT_TEST_TEE_OUTPUT_FILE")")"
- fi
- write_junit_xml_testcase "$1" " $junit_insert"
- fi
+ failure_label=$1
test_failure=$(($test_failure + 1))
say_color error "not ok $test_count - $1"
shift
printf '%s\n' "$*" | sed -e 's/^/# /'
test "$immediate" = "" || _error_exit
+ finalize_test_case_output failure "$failure_label" "$@"
}
test_known_broken_ok_ () {
- if test -n "$write_junit_xml"
- then
- write_junit_xml_testcase "$* (breakage fixed)"
- fi
test_fixed=$(($test_fixed+1))
say_color error "ok $test_count - $@ # TODO known breakage vanished"
+ finalize_test_case_output fixed "$@"
}
test_known_broken_failure_ () {
- if test -n "$write_junit_xml"
- then
- write_junit_xml_testcase "$* (known breakage)"
- fi
test_broken=$(($test_broken+1))
say_color warn "not ok $test_count - $@ # TODO known breakage"
+ finalize_test_case_output broken "$@"
}
test_debug () {
@@ -1081,10 +1063,7 @@ test_start_ () {
test_count=$(($test_count+1))
maybe_setup_verbose
maybe_setup_valgrind
- if test -n "$write_junit_xml"
- then
- junit_start=$(test-tool date getnanos)
- fi
+ start_test_case_output "$@"
}
test_finish_ () {
@@ -1135,15 +1114,10 @@ test_skip () {
case "$to_skip" in
t)
- if test -n "$write_junit_xml"
- then
- message="$(xml_attr_encode "$skipped_reason")"
- write_junit_xml_testcase "$1" \
- " "
- fi
say_color skip "ok $test_count # skip $1 ($skipped_reason)"
: true
+ finalize_test_case_output skip "$@"
;;
*)
false
@@ -1156,53 +1130,6 @@ test_at_end_hook_ () {
:
}
-write_junit_xml () {
- case "$1" in
- --truncate)
- >"$junit_xml_path"
- junit_have_testcase=
- shift
- ;;
- esac
- printf '%s\n' "$@" >>"$junit_xml_path"
-}
-
-xml_attr_encode () {
- printf '%s\n' "$@" | test-tool xml-encode
-}
-
-write_junit_xml_testcase () {
- junit_attrs="name=\"$(xml_attr_encode "$this_test.$test_count $1")\""
- shift
- junit_attrs="$junit_attrs classname=\"$this_test\""
- junit_attrs="$junit_attrs time=\"$(test-tool \
- date getnanos $junit_start)\""
- write_junit_xml "$(printf '%s\n' \
- " " "$@" " ")"
- junit_have_testcase=t
-}
-
-finalize_junit_xml () {
- if test -n "$write_junit_xml" && test -n "$junit_xml_path"
- then
- test -n "$junit_have_testcase" || {
- junit_start=$(test-tool date getnanos)
- write_junit_xml_testcase "all tests skipped"
- }
-
- # adjust the overall time
- junit_time=$(test-tool date getnanos $junit_suite_start)
- sed -e "s/\(]*/& time=\"$junit_time\"/" \
- -e '/^ *<\/testsuite/d' \
- <"$junit_xml_path" >"$junit_xml_path.new"
- mv "$junit_xml_path.new" "$junit_xml_path"
-
- write_junit_xml " " ""
- write_junit_xml=
- fi
-}
-
test_atexit_cleanup=:
test_atexit_handler () {
# In a succeeding test script 'test_atexit_handler' is invoked
@@ -1225,7 +1152,7 @@ test_done () {
# removed, so the commands can access pidfiles and socket files.
test_atexit_handler
- finalize_junit_xml
+ finalize_test_output
if test -z "$HARNESS_ACTIVE"
then
@@ -1516,22 +1443,7 @@ fi
# in subprocesses like git equals our $PWD (for pathname comparisons).
cd -P "$TRASH_DIRECTORY" || exit 1
-if test -n "$write_junit_xml"
-then
- junit_xml_dir="$TEST_OUTPUT_DIRECTORY/out"
- mkdir -p "$junit_xml_dir"
- junit_xml_base=${0##*/}
- junit_xml_path="$junit_xml_dir/TEST-${junit_xml_base%.sh}.xml"
- junit_attrs="name=\"${junit_xml_base%.sh}\""
- junit_attrs="$junit_attrs timestamp=\"$(TZ=UTC \
- date +%Y-%m-%dT%H:%M:%S)\""
- write_junit_xml --truncate "" " "
- junit_suite_start=$(test-tool date getnanos)
- if test -n "$GIT_TEST_TEE_OUTPUT_FILE"
- then
- GIT_TEST_TEE_OFFSET=0
- fi
-fi
+start_test_output "$0"
# Convenience
# A regexp to match 5 and 35 hexdigits