From 1f27522d8c651db5d254d352377051d7071edcdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90o=C3=A0n=20Tr=E1=BA=A7n=20C=C3=B4ng=20Danh?= Date: Wed, 25 Mar 2020 22:06:14 +0700 Subject: [PATCH 01/27] t4061: use POSIX compliant regex(7) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BRE interprets `+` literally, and `\+` is undefined for POSIX BRE, from: https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap09.html#tag_09_03_02 > The interpretation of an ordinary character preceded > by an unescaped ( '\\' ) is undefined, except for: > - The characters ')', '(', '{', and '}' > - The digits 1 to 9 inclusive > - A character inside a bracket expression This test is failing with busybox sed, the default sed of Alpine Linux We have 2 options here: - Using literal `+` because BRE will interpret it as-is, or - Using character class `[+]` to defend against a sed that expects ERE ERE-expected sed is theoretical at this point, but we haven't found it, yet. And, we may run into other problems with that sed. Let's go with first option and fix it later if that sed could be found. Signed-off-by: Đoàn Trần Công Danh Signed-off-by: Junio C Hamano --- t/t4061-diff-indent.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/t4061-diff-indent.sh b/t/t4061-diff-indent.sh index 2affd7a100996f..0f7a6d97a8b7e2 100755 --- a/t/t4061-diff-indent.sh +++ b/t/t4061-diff-indent.sh @@ -17,7 +17,7 @@ compare_diff () { # Compare blame output using the expectation for a diff as reference. # Only look for the lines coming from non-boundary commits. compare_blame () { - sed -n -e "1,4d" -e "s/^\+//p" <"$1" >.tmp-1 + sed -n -e "1,4d" -e "s/^+//p" <"$1" >.tmp-1 sed -ne "s/^[^^][^)]*) *//p" <"$2" >.tmp-2 test_cmp .tmp-1 .tmp-2 && rm -f .tmp-1 .tmp-2 } From 6ec5df61d525fe088ed15951051b17d14a5c243b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90o=C3=A0n=20Tr=E1=BA=A7n=20C=C3=B4ng=20Danh?= Date: Wed, 25 Mar 2020 22:06:15 +0700 Subject: [PATCH 02/27] test-lib-functions: test_cmp: eval $GIT_TEST_CMP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Shell recognises first non-assignment token as command name. With /bin/sh linked to either /bin/bash or /bin/dash, `cd t/perf && ./p0000-perf-lib-sanity.sh -d -i -v` reports: > test_cmp:1: command not found: diff -u Using `eval` to unquote $GIT_TEST_CMP as same as precedence in `git_editor`. Signed-off-by: Đoàn Trần Công Danh Signed-off-by: Junio C Hamano --- t/test-lib-functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/test-lib-functions.sh b/t/test-lib-functions.sh index 352c213d52e2e4..ab0e47ae1748d9 100644 --- a/t/test-lib-functions.sh +++ b/t/test-lib-functions.sh @@ -905,7 +905,7 @@ test_expect_code () { # - not all diff versions understand "-u" test_cmp() { - $GIT_TEST_CMP "$@" + eval "$GIT_TEST_CMP" '"$@"' } # Check that the given config key has the expected value. From 6e45972cd74df4cc18e200b3e9363fe11d6334d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90o=C3=A0n=20Tr=E1=BA=A7n=20C=C3=B4ng=20Danh?= Date: Wed, 25 Mar 2020 22:06:16 +0700 Subject: [PATCH 03/27] t5003: drop the subshell in test_lazy_prereq MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_lazy_prereq will be evaluated in a throw-away directory. Drop unnecessary subshell and mkdir. Signed-off-by: Đoàn Trần Công Danh Signed-off-by: Junio C Hamano --- t/t5003-archive-zip.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/t/t5003-archive-zip.sh b/t/t5003-archive-zip.sh index 106eddbd85b04a..df1374a312d500 100755 --- a/t/t5003-archive-zip.sh +++ b/t/t5003-archive-zip.sh @@ -7,12 +7,8 @@ test_description='git archive --format=zip test' SUBSTFORMAT=%H%n test_lazy_prereq UNZIP_SYMLINKS ' - ( - mkdir unzip-symlinks && - cd unzip-symlinks && - "$GIT_UNZIP" "$TEST_DIRECTORY"/t5003/infozip-symlinks.zip && - test -h symlink - ) + "$GIT_UNZIP" "$TEST_DIRECTORY"/t5003/infozip-symlinks.zip && + test -h symlink ' check_zip() { From ff0dab331e2740f7e565e1539bd5848657766f6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90o=C3=A0n=20Tr=E1=BA=A7n=20C=C3=B4ng=20Danh?= Date: Wed, 25 Mar 2020 22:06:17 +0700 Subject: [PATCH 04/27] t5003: skip conversion test if unzip -a is unavailable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Alpine Linux's default unzip(1) doesn't support `-a`. Skip those tests on that platform. Signed-off-by: Đoàn Trần Công Danh Signed-off-by: Junio C Hamano --- t/t5003-archive-zip.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/t/t5003-archive-zip.sh b/t/t5003-archive-zip.sh index df1374a312d500..3b76d2eb653f7a 100755 --- a/t/t5003-archive-zip.sh +++ b/t/t5003-archive-zip.sh @@ -11,6 +11,10 @@ test_lazy_prereq UNZIP_SYMLINKS ' test -h symlink ' +test_lazy_prereq UNZIP_CONVERT ' + "$GIT_UNZIP" -a "$TEST_DIRECTORY"/t5003/infozip-symlinks.zip +' + check_zip() { zipfile=$1.zip listfile=$1.lst @@ -35,33 +39,33 @@ check_zip() { extracted=${dir_with_prefix}a original=a - test_expect_success UNZIP " extract ZIP archive with EOL conversion" ' + test_expect_success UNZIP_CONVERT " extract ZIP archive with EOL conversion" ' (mkdir $dir && cd $dir && "$GIT_UNZIP" -a ../$zipfile) ' - test_expect_success UNZIP " validate that text files are converted" " + test_expect_success UNZIP_CONVERT " validate that text files are converted" " test_cmp_bin $extracted/text.cr $extracted/text.crlf && test_cmp_bin $extracted/text.cr $extracted/text.lf " - test_expect_success UNZIP " validate that binary files are unchanged" " + test_expect_success UNZIP_CONVERT " validate that binary files are unchanged" " test_cmp_bin $original/binary.cr $extracted/binary.cr && test_cmp_bin $original/binary.crlf $extracted/binary.crlf && test_cmp_bin $original/binary.lf $extracted/binary.lf " - test_expect_success UNZIP " validate that diff files are converted" " + test_expect_success UNZIP_CONVERT " validate that diff files are converted" " test_cmp_bin $extracted/diff.cr $extracted/diff.crlf && test_cmp_bin $extracted/diff.cr $extracted/diff.lf " - test_expect_success UNZIP " validate that -diff files are unchanged" " + test_expect_success UNZIP_CONVERT " validate that -diff files are unchanged" " test_cmp_bin $original/nodiff.cr $extracted/nodiff.cr && test_cmp_bin $original/nodiff.crlf $extracted/nodiff.crlf && test_cmp_bin $original/nodiff.lf $extracted/nodiff.lf " - test_expect_success UNZIP " validate that custom diff is unchanged " " + test_expect_success UNZIP_CONVERT " validate that custom diff is unchanged " " test_cmp_bin $original/custom.cr $extracted/custom.cr && test_cmp_bin $original/custom.crlf $extracted/custom.crlf && test_cmp_bin $original/custom.lf $extracted/custom.lf From 6c28bef2d4bb71ec90be0fa6049d4925f6713903 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90o=C3=A0n=20Tr=E1=BA=A7n=20C=C3=B4ng=20Danh?= Date: Wed, 25 Mar 2020 22:06:18 +0700 Subject: [PATCH 05/27] t5616: use rev-parse instead to get HEAD's object_id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only HEAD's object_id is necessary, rev-list is an overkill. Despite POSIX requires grep(1) treat single pattern with as multiple patterns. busybox's grep(1) (as of v1.31.1) haven't implemented it yet. Use rev-parse to simplify the test and avoid busybox unimplemented features. Signed-off-by: Đoàn Trần Công Danh Signed-off-by: Junio C Hamano --- t/t5616-partial-clone.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/t5616-partial-clone.sh b/t/t5616-partial-clone.sh index 77bb91e9769227..09e640cae40f2a 100755 --- a/t/t5616-partial-clone.sh +++ b/t/t5616-partial-clone.sh @@ -49,7 +49,7 @@ test_expect_success 'do partial clone 1' ' test_expect_success 'verify that .promisor file contains refs fetched' ' ls pc1/.git/objects/pack/pack-*.promisor >promisorlist && test_line_count = 1 promisorlist && - git -C srv.bare rev-list HEAD >headhash && + git -C srv.bare rev-parse --verify HEAD >headhash && grep "$(cat headhash) HEAD" $(cat promisorlist) && grep "$(cat headhash) refs/heads/master" $(cat promisorlist) ' From d51dd4ca3ad12d2d4153418cf13027390748a914 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90o=C3=A0n=20Tr=E1=BA=A7n=20C=C3=B4ng=20Danh?= Date: Wed, 25 Mar 2020 22:06:19 +0700 Subject: [PATCH 06/27] t7063: drop non-POSIX argument "-ls" from find(1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since commit 6b7728db81, (t7063: work around FreeBSD's lazy mtime update feature, 2016-08-03), we started to use ls as a trick to update directory's mtime. However, `-ls` flag isn't required by POSIX's find(1), and busybox(1) doesn't implement it. Use "-exec ls -ld {} +" instead. Signed-off-by: Đoàn Trần Công Danh Signed-off-by: Junio C Hamano --- t/t7063-status-untracked-cache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/t7063-status-untracked-cache.sh b/t/t7063-status-untracked-cache.sh index 190ae149cf3cb6..6738497ea7a851 100755 --- a/t/t7063-status-untracked-cache.sh +++ b/t/t7063-status-untracked-cache.sh @@ -18,7 +18,7 @@ GIT_FORCE_UNTRACKED_CACHE=true export GIT_FORCE_UNTRACKED_CACHE sync_mtime () { - find . -type d -ls >/dev/null + find . -type d -exec ls -ld {} + >/dev/null } avoid_racy() { From f73533aa38e5a8f3e56d1cc7d31852b0710b3e39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90o=C3=A0n=20Tr=E1=BA=A7n=20C=C3=B4ng=20Danh?= Date: Thu, 26 Mar 2020 11:37:37 +0700 Subject: [PATCH 07/27] t4124: tweak test so that non-compliant diff(1) can also be used MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The diff(1) implementation of busybox produces the unified context format even without being asked, and it cannot produce the default format, but a test in this script relies on. We could rewrite the test so that we count the lines in the postimage out of the unified context format, but the format is not supported by some implementations of diff (e.g. HP-UX). Accomodate busybox by adding a fallback code to count postimage lines in unified context output, when counting in the output in the default format finds nothing. Signed-off-by: Đoàn Trần Công Danh [jc: applied Documentation/CodingGuidelines and tweaked the log message] Signed-off-by: Junio C Hamano --- t/t4124-apply-ws-rule.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/t/t4124-apply-ws-rule.sh b/t/t4124-apply-ws-rule.sh index 971a5a7512ac77..0ca29821ece3fd 100755 --- a/t/t4124-apply-ws-rule.sh +++ b/t/t4124-apply-ws-rule.sh @@ -52,6 +52,13 @@ test_fix () { # find touched lines $DIFF file target | sed -n -e "s/^> //p" >fixed + # busybox's diff(1) doesn't output normal format + if ! test -s fixed + then + $DIFF -u file target | + grep -v '^+++ target' | + sed -ne "/^+/s/+//p" >fixed + fi # the changed lines are all expected to change fixed_cnt=$(wc -l Date: Thu, 26 Mar 2020 11:37:38 +0700 Subject: [PATCH 08/27] t5703: feed raw data into test-tool unpack-sideband MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit busybox's sed isn't binary clean. Thus, triggers false-negative on this test. We could replace sed with perl on this usecase. But, we could slightly modify the helper to discard unwanted data in the beginning. Fix the false negative by updating this helper. Helped-by: Jeff King Signed-off-by: Đoàn Trần Công Danh Signed-off-by: Junio C Hamano --- t/helper/test-pkt-line.c | 2 +- t/t5703-upload-pack-ref-in-want.sh | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/t/helper/test-pkt-line.c b/t/helper/test-pkt-line.c index 282d53638446bb..12ca698e17a1d5 100644 --- a/t/helper/test-pkt-line.c +++ b/t/helper/test-pkt-line.c @@ -67,7 +67,7 @@ static void unpack_sideband(void) case PACKET_READ_NORMAL: band = reader.line[0] & 0xff; if (band < 1 || band > 2) - die("unexpected side band %d", band); + continue; /* skip non-sideband packets */ fd = band; write_or_die(fd, reader.line + 1, reader.pktlen - 1); diff --git a/t/t5703-upload-pack-ref-in-want.sh b/t/t5703-upload-pack-ref-in-want.sh index 7fba3063bf9150..a34460f7d82ea0 100755 --- a/t/t5703-upload-pack-ref-in-want.sh +++ b/t/t5703-upload-pack-ref-in-want.sh @@ -13,10 +13,7 @@ get_actual_refs () { } get_actual_commits () { - sed -n -e '/packfile/,/0000/{ - /packfile/d - p - }' o.pack && + test-tool pkt-line unpack-sideband o.pack && git index-pack o.pack && git verify-pack -v o.idx >objs && grep commit objs | cut -d" " -f1 | sort >actual_commits From 120c8b8ba0d251ffdfe9ecc8fedbae9949e9dad8 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 31 Mar 2020 13:38:52 +0200 Subject: [PATCH 09/27] README: remove obsolete build badge We no longer test specifically whether Git's source code builds under `core.autoCRLF=true`, as this is already tested implicitly by our Windows CI/PR builds. Signed-off-by: Johannes Schindelin --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 90e0feef3cf7fd..8a7aee2663f868 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,6 @@ Git for Windows =============== [![Build Status (Windows/macOS/Linux)](https://dev.azure.com/git-for-windows/git/_apis/build/status/git-for-windows.git)](https://dev.azure.com/git-for-windows/git/_build/latest?definitionId=17) -[![Build Status (core.autocrlf=true)](https://dev.azure.com/Git-for-Windows/git/_apis/build/status/TestWithAutoCRLF)](https://dev.azure.com/Git-for-Windows/git/_build/latest?definitionId=3) [![Join the chat at https://gitter.im/git-for-windows/git](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/git-for-windows/git?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) This is [Git for Windows](http://git-for-windows.github.io/), the Windows port From 8023d01bcc93cbf06a98e7990316a42ba3412370 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?SZEDER=20G=C3=A1bor?= Date: Wed, 1 Apr 2020 23:21:51 +0200 Subject: [PATCH 10/27] ci: make MAKEFLAGS available inside the Docker container in the Linux32 job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Once upon a time we ran 'make --jobs=2 ...' to build Git, its documentation, or to apply Coccinelle semantic patches. Then commit eaa62291ff (ci: inherit --jobs via MAKEFLAGS in run-build-and-tests, 2019-01-27) came along, and started using the MAKEFLAGS environment variable to centralize setting the number of parallel jobs in 'ci/libs.sh'. Alas, it forgot to update 'ci/run-linux32-docker.sh' to make MAKEFLAGS available inside the Docker container running the 32 bit Linux job, and, consequently, since then that job builds Git sequentially, and it ignores any Makefile knobs that we might set in MAKEFLAGS (though we don't set any for the 32 bit Linux job at the moment). So update the 'docker run' invocation in 'ci/run-linux32-docker.sh' to make MAKEFLAGS available inside the Docker container as well. Set CC=gcc for the 32 bit Linux job, because that's the compiler installed in the 32 bit Linux Docker image that we use (Travis CI nowadays sets CC=clang by default, but clang is not installed in this image). Signed-off-by: SZEDER Gábor Signed-off-by: Đoàn Trần Công Danh --- ci/lib.sh | 3 +++ ci/run-linux32-docker.sh | 1 + 2 files changed, 4 insertions(+) diff --git a/ci/lib.sh b/ci/lib.sh index a90d0dc0fd2ae3..804833528148ea 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -192,6 +192,9 @@ osx-clang|osx-gcc) GIT_TEST_GETTEXT_POISON) export GIT_TEST_GETTEXT_POISON=true ;; +Linux32) + CC=gcc + ;; esac MAKEFLAGS="$MAKEFLAGS CC=${CC:-cc}" diff --git a/ci/run-linux32-docker.sh b/ci/run-linux32-docker.sh index 751acfcf8a8c5c..ebb18fa7472549 100755 --- a/ci/run-linux32-docker.sh +++ b/ci/run-linux32-docker.sh @@ -20,6 +20,7 @@ docker run \ --env GIT_PROVE_OPTS \ --env GIT_TEST_OPTS \ --env GIT_TEST_CLONE_2GB \ + --env MAKEFLAGS \ --env cache_dir="$container_cache_dir" \ --volume "${PWD}:/usr/src/git" \ --volume "$cache_dir:$container_cache_dir" \ From 9a6c71177b3b71adbcbd7368d2ddc7c314c00d5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90o=C3=A0n=20Tr=E1=BA=A7n=20C=C3=B4ng=20Danh?= Date: Thu, 2 Apr 2020 12:55:10 +0700 Subject: [PATCH 11/27] ci/lib-docker: preserve required environment variables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We're using "su -m" to preserve environment variables in the shell run by "su". But, that options will be ignored while "-l" (aka "--login") is specified in util-linux and busybox's su. In a later patch this script will be reused for checking Git for Linux with musl libc on Alpine Linux, Alpine Linux uses "su" from busybox. Since we don't have interest in all environment variables, pass only those necessary variables to the inner script. Signed-off-by: Đoàn Trần Công Danh --- ci/run-linux32-build.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/ci/run-linux32-build.sh b/ci/run-linux32-build.sh index e3a193adbce39c..7f985615c29081 100755 --- a/ci/run-linux32-build.sh +++ b/ci/run-linux32-build.sh @@ -51,10 +51,17 @@ else fi # Build and test -linux32 --32bit i386 su -m -l $CI_USER -c ' +linux32 --32bit i386 su -m -l $CI_USER -c " set -ex + export DEVELOPER='$DEVELOPER' + export DEFAULT_TEST_TARGET='$DEFAULT_TEST_TARGET' + export GIT_PROVE_OPTS='$GIT_PROVE_OPTS' + export GIT_TEST_OPTS='$GIT_TEST_OPTS' + export GIT_TEST_CLONE_2GB='$GIT_TEST_CLONE_2GB' + export MAKEFLAGS='$MAKEFLAGS' + export cache_dir='$cache_dir' cd /usr/src/git - test -n "$cache_dir" && ln -s "$cache_dir/.prove" t/.prove + test -n '$cache_dir' && ln -s '$cache_dir/.prove' t/.prove make make test -' +" From 61f96e27c3159bcdefd04578655598c42c97cbf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90o=C3=A0n=20Tr=E1=BA=A7n=20C=C3=B4ng=20Danh?= Date: Thu, 2 Apr 2020 13:42:25 +0700 Subject: [PATCH 12/27] ci/linux32: parameterise command to switch arch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In a later patch, the remaining of this command will be re-used for the CI job for linux with musl libc. Allow customisation of the emulator, now. Signed-off-by: Đoàn Trần Công Danh --- ci/run-linux32-build.sh | 13 +++++++++++-- ci/run-linux32-docker.sh | 2 ++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/ci/run-linux32-build.sh b/ci/run-linux32-build.sh index 7f985615c29081..44bb332f648744 100755 --- a/ci/run-linux32-build.sh +++ b/ci/run-linux32-build.sh @@ -14,8 +14,17 @@ then exit 1 fi +case "$jobname" in +Linux32) + switch_cmd="linux32 --32bit i386" + ;; +*) + exit 1 + ;; +esac + # Update packages to the latest available versions -linux32 --32bit i386 sh -c ' +command $switch_cmd sh -c ' apt update >/dev/null && apt install -y build-essential libcurl4-openssl-dev libssl-dev \ libexpat-dev gettext python >/dev/null @@ -51,7 +60,7 @@ else fi # Build and test -linux32 --32bit i386 su -m -l $CI_USER -c " +command $switch_cmd su -m -l $CI_USER -c " set -ex export DEVELOPER='$DEVELOPER' export DEFAULT_TEST_TARGET='$DEFAULT_TEST_TARGET' diff --git a/ci/run-linux32-docker.sh b/ci/run-linux32-docker.sh index ebb18fa7472549..54186b6aa7eb87 100755 --- a/ci/run-linux32-docker.sh +++ b/ci/run-linux32-docker.sh @@ -9,6 +9,7 @@ docker pull daald/ubuntu32:xenial # Use the following command to debug the docker build locally: # $ docker run -itv "${PWD}:/usr/src/git" --entrypoint /bin/bash daald/ubuntu32:xenial +# root@container:/# export jobname= # root@container:/# /usr/src/git/ci/run-linux32-build.sh container_cache_dir=/tmp/travis-cache @@ -21,6 +22,7 @@ docker run \ --env GIT_TEST_OPTS \ --env GIT_TEST_CLONE_2GB \ --env MAKEFLAGS \ + --env jobname \ --env cache_dir="$container_cache_dir" \ --volume "${PWD}:/usr/src/git" \ --volume "$cache_dir:$container_cache_dir" \ From 53170f395002f106a3cc89f39812f0e594d4e0a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90o=C3=A0n=20Tr=E1=BA=A7n=20C=C3=B4ng=20Danh?= Date: Thu, 26 Mar 2020 14:35:18 +0700 Subject: [PATCH 13/27] ci: refactor docker runner script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We will support alpine check in docker later in this series. While we're at it, tell people to run as root in podman, if podman is used as drop-in replacement for docker, because podman will map host-user to container's root, therefore, mapping their permission. Signed-off-by: Đoàn Trần Công Danh --- .travis.yml | 2 +- azure-pipelines.yml | 4 ++-- ...n-linux32-build.sh => run-docker-build.sh} | 6 ++--- ci/{run-linux32-docker.sh => run-docker.sh} | 22 ++++++++++++++----- 4 files changed, 22 insertions(+), 12 deletions(-) rename ci/{run-linux32-build.sh => run-docker-build.sh} (93%) rename ci/{run-linux32-docker.sh => run-docker.sh} (53%) diff --git a/.travis.yml b/.travis.yml index fc5730b085f117..069aeeff3c96eb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,7 +32,7 @@ matrix: services: - docker before_install: - script: ci/run-linux32-docker.sh + script: ci/run-docker.sh - env: jobname=StaticAnalysis os: linux compiler: diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 675c3a43c9cc2b..11413f66f89662 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -489,14 +489,14 @@ jobs: test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 res=0 - sudo AGENT_OS="$AGENT_OS" BUILD_BUILDNUMBER="$BUILD_BUILDNUMBER" BUILD_REPOSITORY_URI="$BUILD_REPOSITORY_URI" BUILD_SOURCEBRANCH="$BUILD_SOURCEBRANCH" BUILD_SOURCEVERSION="$BUILD_SOURCEVERSION" SYSTEM_PHASENAME="$SYSTEM_PHASENAME" SYSTEM_TASKDEFINITIONSURI="$SYSTEM_TASKDEFINITIONSURI" SYSTEM_TEAMPROJECT="$SYSTEM_TEAMPROJECT" CC=$CC MAKEFLAGS="$MAKEFLAGS" bash -lxc ci/run-linux32-docker.sh || res=1 + sudo AGENT_OS="$AGENT_OS" BUILD_BUILDNUMBER="$BUILD_BUILDNUMBER" BUILD_REPOSITORY_URI="$BUILD_REPOSITORY_URI" BUILD_SOURCEBRANCH="$BUILD_SOURCEBRANCH" BUILD_SOURCEVERSION="$BUILD_SOURCEVERSION" SYSTEM_PHASENAME="$SYSTEM_PHASENAME" SYSTEM_TASKDEFINITIONSURI="$SYSTEM_TASKDEFINITIONSURI" SYSTEM_TEAMPROJECT="$SYSTEM_TEAMPROJECT" CC=$CC MAKEFLAGS="$MAKEFLAGS" jobname=Linux32 bash -lxc ci/run-docker.sh || res=1 sudo chmod a+r t/out/TEST-*.xml test ! -d t/failed-test-artifacts || sudo chmod a+r t/failed-test-artifacts test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || sudo umount "$HOME/test-cache" || res=1 exit $res - displayName: 'ci/run-linux32-docker.sh' + displayName: 'jobname=Linux32 ci/run-docker.sh' env: GITFILESHAREPWD: $(gitfileshare.pwd) - task: PublishTestResults@2 diff --git a/ci/run-linux32-build.sh b/ci/run-docker-build.sh similarity index 93% rename from ci/run-linux32-build.sh rename to ci/run-docker-build.sh index 44bb332f648744..a05b48c5591530 100755 --- a/ci/run-linux32-build.sh +++ b/ci/run-docker-build.sh @@ -1,16 +1,16 @@ #!/bin/sh # -# Build and test Git in a 32-bit environment +# Build and test Git inside container # # Usage: -# run-linux32-build.sh +# run-docker-build.sh # set -ex if test $# -ne 1 || test -z "$1" then - echo >&2 "usage: run-linux32-build.sh " + echo >&2 "usage: run-docker-build.sh " exit 1 fi diff --git a/ci/run-linux32-docker.sh b/ci/run-docker.sh similarity index 53% rename from ci/run-linux32-docker.sh rename to ci/run-docker.sh index 54186b6aa7eb87..3881f99b533017 100755 --- a/ci/run-linux32-docker.sh +++ b/ci/run-docker.sh @@ -1,16 +1,26 @@ #!/bin/sh # -# Download and run Docker image to build and test 32-bit Git +# Download and run Docker image to build and test Git # . ${0%/*}/lib.sh -docker pull daald/ubuntu32:xenial +case "$jobname" in +Linux32) + CI_CONTAINER="daald/ubuntu32:xenial" + ;; +*) + exit 1 + ;; +esac + +docker pull "$CI_CONTAINER" # Use the following command to debug the docker build locally: -# $ docker run -itv "${PWD}:/usr/src/git" --entrypoint /bin/bash daald/ubuntu32:xenial +# must be 0 if podman is used as drop-in replacement for docker +# $ docker run -itv "${PWD}:/usr/src/git" --entrypoint /bin/sh "$CI_CONTAINER" # root@container:/# export jobname= -# root@container:/# /usr/src/git/ci/run-linux32-build.sh +# root@container:/# /usr/src/git/ci/run-docker-build.sh container_cache_dir=/tmp/travis-cache @@ -26,8 +36,8 @@ docker run \ --env cache_dir="$container_cache_dir" \ --volume "${PWD}:/usr/src/git" \ --volume "$cache_dir:$container_cache_dir" \ - daald/ubuntu32:xenial \ - /usr/src/git/ci/run-linux32-build.sh $(id -u $USER) + "$CI_CONTAINER" \ + /usr/src/git/ci/run-docker-build.sh $(id -u $USER) check_unignored_build_artifacts From 3c781e3c27177a46f8b4925b36834de717b02521 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90o=C3=A0n=20Tr=E1=BA=A7n=20C=C3=B4ng=20Danh?= Date: Thu, 2 Apr 2020 13:35:30 +0700 Subject: [PATCH 14/27] ci/linux32: libify install-dependencies step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In a later patch, we will add new Travis Job for linux-musl. Most of other code in this file could be reuse for that job. Move the code to install dependencies to a common script. Should we add new CI system that can run directly in container, we can reuse this script for installation step. Signed-off-by: Đoàn Trần Công Danh --- ci/install-docker-dependencies.sh | 14 ++++++++++++++ ci/run-docker-build.sh | 7 +------ 2 files changed, 15 insertions(+), 6 deletions(-) create mode 100755 ci/install-docker-dependencies.sh diff --git a/ci/install-docker-dependencies.sh b/ci/install-docker-dependencies.sh new file mode 100755 index 00000000000000..a104c61d2925d3 --- /dev/null +++ b/ci/install-docker-dependencies.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# +# Install dependencies required to build and test Git inside container +# + +case "$jobname" in +Linux32) + linux32 --32bit i386 sh -c ' + apt update >/dev/null && + apt install -y build-essential libcurl4-openssl-dev \ + libssl-dev libexpat-dev gettext python >/dev/null + ' + ;; +esac diff --git a/ci/run-docker-build.sh b/ci/run-docker-build.sh index a05b48c5591530..4a153492ba29b2 100755 --- a/ci/run-docker-build.sh +++ b/ci/run-docker-build.sh @@ -23,12 +23,7 @@ Linux32) ;; esac -# Update packages to the latest available versions -command $switch_cmd sh -c ' - apt update >/dev/null && - apt install -y build-essential libcurl4-openssl-dev libssl-dev \ - libexpat-dev gettext python >/dev/null -' +"${0%/*}/install-docker-dependencies.sh" # If this script runs inside a docker container, then all commands are # usually executed as root. Consequently, the host user might not be From 396481a12779315852737e42f242bc1f5fb1e88f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90o=C3=A0n=20Tr=E1=BA=A7n=20C=C3=B4ng=20Danh?= Date: Thu, 26 Mar 2020 14:35:19 +0700 Subject: [PATCH 15/27] travis: build and test on Linux with musl libc and busybox MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Đoàn Trần Công Danh --- .travis.yml | 8 ++++++++ ci/install-docker-dependencies.sh | 4 ++++ ci/lib.sh | 5 +++++ ci/run-docker-build.sh | 4 ++++ ci/run-docker.sh | 3 +++ 5 files changed, 24 insertions(+) diff --git a/.travis.yml b/.travis.yml index 069aeeff3c96eb..0cfc3c3428bd37 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,6 +33,14 @@ matrix: - docker before_install: script: ci/run-docker.sh + - env: jobname=linux-musl + os: linux + compiler: + addons: + services: + - docker + before_install: + script: ci/run-docker.sh - env: jobname=StaticAnalysis os: linux compiler: diff --git a/ci/install-docker-dependencies.sh b/ci/install-docker-dependencies.sh index a104c61d2925d3..26a6689766d7f1 100755 --- a/ci/install-docker-dependencies.sh +++ b/ci/install-docker-dependencies.sh @@ -11,4 +11,8 @@ Linux32) libssl-dev libexpat-dev gettext python >/dev/null ' ;; +linux-musl) + apk add --update build-base curl-dev openssl-dev expat-dev gettext \ + pcre2-dev python3 musl-libintl perl-utils ncurses >/dev/null + ;; esac diff --git a/ci/lib.sh b/ci/lib.sh index 804833528148ea..e9a5c51425a25a 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -195,6 +195,11 @@ GIT_TEST_GETTEXT_POISON) Linux32) CC=gcc ;; +linux-musl) + CC=gcc + MAKEFLAGS="$MAKEFLAGS PYTHON_PATH=/usr/bin/python3 USE_LIBPCRE2=Yes" + MAKEFLAGS="$MAKEFLAGS NO_REGEX=Yes ICONV_OMITS_BOM=Yes" + ;; esac MAKEFLAGS="$MAKEFLAGS CC=${CC:-cc}" diff --git a/ci/run-docker-build.sh b/ci/run-docker-build.sh index 4a153492ba29b2..8d47a5fda3b1c9 100755 --- a/ci/run-docker-build.sh +++ b/ci/run-docker-build.sh @@ -18,6 +18,10 @@ case "$jobname" in Linux32) switch_cmd="linux32 --32bit i386" ;; +linux-musl) + switch_cmd= + useradd () { adduser -D "$@"; } + ;; *) exit 1 ;; diff --git a/ci/run-docker.sh b/ci/run-docker.sh index 3881f99b533017..37fa372052ddb8 100755 --- a/ci/run-docker.sh +++ b/ci/run-docker.sh @@ -9,6 +9,9 @@ case "$jobname" in Linux32) CI_CONTAINER="daald/ubuntu32:xenial" ;; +linux-musl) + CI_CONTAINER=alpine + ;; *) exit 1 ;; From e56f24aec457130a855fa0955ad30e2fe1ccc66f Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 31 Mar 2020 12:48:29 +0000 Subject: [PATCH 16/27] ci/lib: if CI type is unknown, show the environment variables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This should help with adding new CI-specific if-else arms. Signed-off-by: Johannes Schindelin Signed-off-by: Đoàn Trần Công Danh --- ci/lib.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/lib.sh b/ci/lib.sh index e9a5c51425a25a..05fbcf681aae00 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -138,6 +138,7 @@ then GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS" else echo "Could not identify CI type" >&2 + env >&2 exit 1 fi From 290de522d967e8072f06be850e3ae3712aa40007 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 7 Apr 2020 09:31:25 +0700 Subject: [PATCH 17/27] ci/lib: allow running in GitHub Actions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For each CI system we support, we need a specific arm in that if/else construct in ci/lib.sh. Let's add one for GitHub Actions. Signed-off-by: Johannes Schindelin Signed-off-by: Đoàn Trần Công Danh --- ci/lib.sh | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/ci/lib.sh b/ci/lib.sh index 05fbcf681aae00..a21c6241b092be 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -34,7 +34,7 @@ save_good_tree () { # successfully before (e.g. because the branch got rebased, changing only # the commit messages). skip_good_tree () { - if test "$TRAVIS_DEBUG_MODE" = true + if test "$TRAVIS_DEBUG_MODE" = true || test true = "$GITHUB_ACTIONS" then return fi @@ -136,6 +136,24 @@ then MAKEFLAGS="$MAKEFLAGS --jobs=10" test windows_nt != "$CI_OS_NAME" || GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS" +elif test true = "$GITHUB_ACTIONS" +then + CI_TYPE=github-actions + CI_BRANCH="$GITHUB_REF" + CI_COMMIT="$GITHUB_SHA" + CI_OS_NAME="$(echo "$RUNNER_OS" | tr A-Z a-z)" + test macos != "$CI_OS_NAME" || CI_OS_NAME=osx + CI_REPO_SLUG="$GITHUB_REPOSITORY" + CI_JOB_ID="$GITHUB_RUN_ID" + CC="${CC:-gcc}" + + cache_dir="$HOME/none" + + export GIT_PROVE_OPTS="--timer --jobs 10" + export GIT_TEST_OPTS="--verbose-log -x" + MAKEFLAGS="$MAKEFLAGS --jobs=10" + test windows != "$CI_OS_NAME" || + GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS" else echo "Could not identify CI type" >&2 env >&2 From c66945b1ffbd826277c2ea1588b4b967a98040c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90o=C3=A0n=20Tr=E1=BA=A7n=20C=C3=B4ng=20Danh?= Date: Tue, 7 Apr 2020 09:31:26 +0700 Subject: [PATCH 18/27] ci/lib: set TERM environment variable if not exist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GitHub Action doesn't set TERM environment variable, which is required by "tput". Fallback to dumb if it's not set. Signed-off-by: Đoàn Trần Công Danh --- ci/lib.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/lib.sh b/ci/lib.sh index a21c6241b092be..5c20975c838077 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -79,6 +79,9 @@ check_unignored_build_artifacts () } } +# GitHub Action doesn't set TERM, which is required by tput +export TERM=${TERM:-dumb} + # Clear MAKEFLAGS that may come from the outside world. export MAKEFLAGS= From 25edca25c76440a7d04a267669bee4e6a774b595 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 7 Apr 2020 09:31:27 +0700 Subject: [PATCH 19/27] ci: fix the `jobname` of the `GETTEXT_POISON` job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In 6cdccfce1e0f (i18n: make GETTEXT_POISON a runtime option, 2018-11-08), the `jobname` was adjusted to have the `GIT_TEST_` prefix, but that prefix makes no sense in this context. Co-authored-by: Đoàn Trần Công Danh Signed-off-by: Johannes Schindelin Signed-off-by: Đoàn Trần Công Danh --- .travis.yml | 2 +- ci/lib.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0cfc3c3428bd37..05f3e3f8d79117 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,7 +16,7 @@ compiler: matrix: include: - - env: jobname=GIT_TEST_GETTEXT_POISON + - env: jobname=GETTEXT_POISON os: linux compiler: addons: diff --git a/ci/lib.sh b/ci/lib.sh index 5c20975c838077..8b39624f3cc462 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -211,7 +211,7 @@ osx-clang|osx-gcc) # Travis CI OS X export GIT_SKIP_TESTS="t9810 t9816" ;; -GIT_TEST_GETTEXT_POISON) +GETTEXT_POISON) export GIT_TEST_GETTEXT_POISON=true ;; Linux32) From e81d6f7607d27b35df255d98998f43b1b8163292 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90o=C3=A0n=20Tr=E1=BA=A7n=20C=C3=B4ng=20Danh?= Date: Tue, 7 Apr 2020 09:31:29 +0700 Subject: [PATCH 20/27] ci: explicit install all required packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In a later patch, we will support GitHub Action. Explicitly install all of our build dependencies. Since GitHub Action VM hasn't install our build dependencies, yet. And there're no harm to reinstall them (in Travis) Signed-off-by: Đoàn Trần Công Danh --- ci/install-dependencies.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index 497fd32ca837a9..371902bb759fa5 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -7,12 +7,17 @@ P4WHENCE=http://filehost.perforce.com/perforce/r$LINUX_P4_VERSION LFSWHENCE=https://github.com/github/git-lfs/releases/download/v$LINUX_GIT_LFS_VERSION +UBUNTU_COMMON_PKGS="make libssl-dev libcurl4-openssl-dev libexpat-dev + perl-modules liberror-perl tcl tk gettext zlib1g-dev apache2 + libauthen-sasl-perl libemail-valid-perl libio-socket-ssl-perl + libnet-smtp-ssl-perl" case "$jobname" in linux-clang|linux-gcc) sudo apt-add-repository -y "ppa:ubuntu-toolchain-r/test" sudo apt-get -q update - sudo apt-get -q -y install language-pack-is libsvn-perl apache2 + sudo apt-get -q -y install language-pack-is libsvn-perl apache2 \ + $UBUNTU_COMMON_PKGS case "$jobname" in linux-gcc) sudo apt-get -q -y install gcc-8 @@ -63,11 +68,16 @@ StaticAnalysis) ;; Documentation) sudo apt-get -q update - sudo apt-get -q -y install asciidoc xmlto docbook-xsl-ns + sudo apt-get -q -y install asciidoc xmlto docbook-xsl-ns \ + libcurl4-openssl-dev test -n "$ALREADY_HAVE_ASCIIDOCTOR" || gem install --version 1.5.8 asciidoctor ;; +GETTEXT_POISON) + sudo apt-get -q update + sudo apt-get -q -y install $UBUNTU_COMMON_PKGS + ;; esac if type p4d >/dev/null && type p4 >/dev/null From 7bc64fa7d239fd4ba82cbfd82f92bf4fb017dfb0 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 7 Apr 2020 09:31:30 +0700 Subject: [PATCH 21/27] ci: run gem with sudo to install asciidoctor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In a later patch, we will run Documentation job in GitHub Actions. The job will run without elevated permission. Run `gem` with `sudo` to elevate permission in order to be able to install to system location. This will also keep this installation in-line with other installation in our Linux system for CI. Signed-off-by: Johannes Schindelin [Danh: reword commit message] Signed-off-by: Đoàn Trần Công Danh --- ci/install-dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index 371902bb759fa5..a6a1dafa154187 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -72,7 +72,7 @@ Documentation) libcurl4-openssl-dev test -n "$ALREADY_HAVE_ASCIIDOCTOR" || - gem install --version 1.5.8 asciidoctor + sudo gem install --version 1.5.8 asciidoctor ;; GETTEXT_POISON) sudo apt-get -q update From 29887869776f09814daaec3980c8b4cf7bda6714 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=90o=C3=A0n=20Tr=E1=BA=A7n=20C=C3=B4ng=20Danh?= Date: Tue, 7 Apr 2020 09:31:31 +0700 Subject: [PATCH 22/27] ci: configure GitHub Actions for CI/PR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch adds CI builds via GitHub Actions. While the underlying technology is at least _very_ similar to that of Azure Pipelines, GitHub Actions are much easier to set up than Azure Pipelines: - no need to install a GitHub App, - no need to set up an Azure DevOps account, - all you need to do is push to your fork on GitHub. Therefore, it makes a lot of sense for us to have a working GitHub Actions setup. While copy/editing `azure-pipelines.yml` into `.github/workflows/main.yml`, we also use the opportunity to accelerate the step that sets up a minimal subset of Git for Windows' SDK in the Windows-build job: - we now download a `.tar.xz` stored in Azure Blobs and extract it simultaneously by calling `curl` and piping the result to `tar`, - decompressing via `xz`, - all three utilities are installed together with Git for Windows At the same time, we also make use of the matrix build feature, which reduces the amount of repeated text by quite a bit. Also, we do away with the parts that try to mount a file share on which `prove` can store data between runs. It is just too complicated to set up, and most times the tree changes anyway, so there is little return on investment there. Initial-patch-by: Johannes Schindelin Signed-off-by: Johannes Schindelin Signed-off-by: Đoàn Trần Công Danh --- .github/workflows/main.yml | 212 +++++++++++++++++++++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 00000000000000..e1ac6d23b4c467 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,212 @@ +name: CI/PR + +on: [push, pull_request] + +env: + DEVELOPER: 1 + +jobs: + windows-build: + runs-on: windows-latest + steps: + - uses: actions/checkout@v1 + - name: download git-sdk-64-minimal + shell: bash + run: a=git-sdk-64-minimal && mkdir -p $a && curl -# https://wingit.blob.core.windows.net/ci-artifacts/$a.tar.xz | tar -C $a -xJf - + - name: build + shell: powershell + env: + HOME: ${{runner.workspace}} + MSYSTEM: MINGW64 + NO_PERL: 1 + run: | + & .\git-sdk-64-minimal\usr\bin\bash.exe -lc @" + printf '%s\n' /git-sdk-64-minimal/ >>.git/info/exclude + + ci/make-test-artifacts.sh artifacts + "@ + - name: upload build artifacts + uses: actions/upload-artifact@v1 + with: + name: windows-artifacts + path: artifacts + windows-test: + runs-on: windows-latest + needs: [windows-build] + strategy: + matrix: + nr: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + steps: + - uses: actions/checkout@v1 + - name: download git-sdk-64-minimal + shell: bash + run: a=git-sdk-64-minimal && mkdir -p $a && curl -# https://wingit.blob.core.windows.net/ci-artifacts/$a.tar.xz | tar -C $a -xJf - + - name: download build artifacts + uses: actions/download-artifact@v1 + with: + name: windows-artifacts + path: ${{github.workspace}} + - name: extract build artifacts + shell: bash + run: tar xf artifacts.tar.gz + - name: test + shell: powershell + run: | + & .\git-sdk-64-minimal\usr\bin\bash.exe -lc @" + # Let Git ignore the SDK + printf '%s\n' /git-sdk-64-minimal/ >>.git/info/exclude + + ci/run-test-slice.sh ${{matrix.nr}} 10 + "@ + - name: ci/print-test-failures.sh + if: failure() + shell: powershell + run: | + & .\git-sdk-64-minimal\usr\bin\bash.exe -lc ci/print-test-failures.sh + vs-build: + env: + MSYSTEM: MINGW64 + NO_PERL: 1 + GIT_CONFIG_PARAMETERS: "'user.name=CI' 'user.email=ci@git'" + runs-on: windows-latest + steps: + - uses: actions/checkout@v1 + - name: download git-sdk-64-minimal + shell: bash + run: a=git-sdk-64-minimal && mkdir -p $a && curl -# https://wingit.blob.core.windows.net/ci-artifacts/$a.tar.xz | tar -C $a -xJf - + - name: generate Visual Studio solution + shell: powershell + run: | + & .\git-sdk-64-minimal\usr\bin\bash.exe -lc @" + make NDEBUG=1 DEVELOPER=1 vcxproj + "@ + if (!$?) { exit(1) } + - name: download vcpkg artifacts + shell: powershell + run: | + $urlbase = "https://dev.azure.com/git/git/_apis/build/builds" + $id = ((Invoke-WebRequest -UseBasicParsing "${urlbase}?definitions=9&statusFilter=completed&resultFilter=succeeded&`$top=1").content | ConvertFrom-JSON).value[0].id + $downloadUrl = ((Invoke-WebRequest -UseBasicParsing "${urlbase}/$id/artifacts").content | ConvertFrom-JSON).value[0].resource.downloadUrl + (New-Object Net.WebClient).DownloadFile($downloadUrl, "compat.zip") + Expand-Archive compat.zip -DestinationPath . -Force + Remove-Item compat.zip + - name: add msbuild to PATH + uses: microsoft/setup-msbuild@v1.0.0 + - name: MSBuild + run: msbuild git.sln -property:Configuration=Release -property:Platform=x64 -maxCpuCount:4 -property:PlatformToolset=v142 + - name: bundle artifact tar + shell: powershell + env: + MSVC: 1 + VCPKG_ROOT: ${{github.workspace}}\compat\vcbuild\vcpkg + run: | + & compat\vcbuild\vcpkg_copy_dlls.bat release + if (!$?) { exit(1) } + & git-sdk-64-minimal\usr\bin\bash.exe -lc @" + mkdir -p artifacts && + eval \"`$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts 2>&1 | grep ^tar)\" + "@ + - name: upload build artifacts + uses: actions/upload-artifact@v1 + with: + name: vs-artifacts + path: artifacts + vs-test: + runs-on: windows-latest + needs: [vs-build] + strategy: + matrix: + nr: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + steps: + - uses: actions/checkout@v1 + - name: download git-64-portable + shell: bash + run: a=git-64-portable && mkdir -p $a && curl -# https://wingit.blob.core.windows.net/ci-artifacts/$a.tar.xz | tar -C $a -xJf - + - name: download build artifacts + uses: actions/download-artifact@v1 + with: + name: vs-artifacts + path: ${{github.workspace}} + - name: extract build artifacts + shell: bash + run: tar xf artifacts.tar.gz + - name: test (parallel) + shell: powershell + env: + MSYSTEM: MINGW64 + NO_SVN_TESTS: 1 + GIT_TEST_SKIP_REBASE_P: 1 + run: | + & git-64-portable\git-cmd.exe --command=usr\bin\bash.exe -lc @" + # Let Git ignore the SDK and the test-cache + printf '%s\n' /git-64-portable/ /test-cache/ >>.git/info/exclude + + cd t && + PATH=\"`$PWD/helper:`$PATH\" && + test-tool.exe run-command testsuite --jobs=10 -V -x --write-junit-xml \ + `$(test-tool.exe path-utils slice-tests \ + ${{matrix.nr}} 10 t[0-9]*.sh) + "@ + regular: + strategy: + matrix: + vector: + - jobname: linux-clang + cc: clang + pool: ubuntu-latest + - jobname: linux-gcc + cc: gcc + pool: ubuntu-latest + - jobname: osx-clang + cc: clang + pool: macos-latest + - jobname: osx-gcc + cc: gcc + pool: macos-latest + - jobname: GETTEXT_POISON + cc: gcc + pool: ubuntu-latest + env: + CC: ${{matrix.vector.cc}} + jobname: ${{matrix.vector.jobname}} + runs-on: ${{matrix.vector.pool}} + steps: + - uses: actions/checkout@v1 + - run: ci/install-dependencies.sh + - run: ci/run-build-and-tests.sh + - run: ci/print-test-failures.sh + if: failure() + dockerized: + strategy: + matrix: + vector: + - jobname: linux-musl + image: alpine + - jobname: Linux32 + image: daald/ubuntu32:xenial + env: + jobname: ${{matrix.vector.jobname}} + runs-on: ubuntu-latest + container: ${{matrix.vector.image}} + steps: + - uses: actions/checkout@v1 + - run: ci/install-docker-dependencies.sh + - run: ci/run-build-and-tests.sh + - run: ci/print-test-failures.sh + if: failure() + static-analysis: + env: + jobname: StaticAnalysis + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - run: ci/install-dependencies.sh + - run: ci/run-static-analysis.sh + documentation: + env: + jobname: Documentation + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - run: ci/install-dependencies.sh + - run: ci/test-documentation.sh From 9ebd9fcbc9cf40a561ba0708e6be7b1856b34362 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 7 Apr 2020 09:31:32 +0700 Subject: [PATCH 23/27] README: add a build badge for the GitHub Actions runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Johannes Schindelin Signed-off-by: Đoàn Trần Công Danh --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9d4564c8aa19cc..e2e00ae24952da 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ +[![Build status](https://github.com/git/git/workflows/CI/PR/badge.svg)](https://github.com/git/git/actions?query=branch%3Amaster+event%3Apush) [![Build Status](https://dev.azure.com/git/git/_apis/build/status/git.git)](https://dev.azure.com/git/git/_build/latest?definitionId=11) Git - fast, scalable, distributed revision control system From 0c8ca7efcaa5e0be8814034a61952bbc5b662301 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 7 Apr 2020 09:31:33 +0700 Subject: [PATCH 24/27] ci: retire the Azure Pipelines definition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We have GitHub Actions now. Running the same builds and tests in Azure Pipelines would be redundant, and a waste of energy. Signed-off-by: Johannes Schindelin Signed-off-by: Đoàn Trần Công Danh --- README.md | 1 - azure-pipelines.yml | 558 -------------------------------------------- 2 files changed, 559 deletions(-) delete mode 100644 azure-pipelines.yml diff --git a/README.md b/README.md index e2e00ae24952da..eb8115e6b04814 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ [![Build status](https://github.com/git/git/workflows/CI/PR/badge.svg)](https://github.com/git/git/actions?query=branch%3Amaster+event%3Apush) -[![Build Status](https://dev.azure.com/git/git/_apis/build/status/git.git)](https://dev.azure.com/git/git/_build/latest?definitionId=11) Git - fast, scalable, distributed revision control system ========================================================= diff --git a/azure-pipelines.yml b/azure-pipelines.yml deleted file mode 100644 index 11413f66f89662..00000000000000 --- a/azure-pipelines.yml +++ /dev/null @@ -1,558 +0,0 @@ -variables: - Agent.Source.Git.ShallowFetchDepth: 1 - -jobs: -- job: windows_build - displayName: Windows Build - condition: succeeded() - pool: - vmImage: windows-latest - timeoutInMinutes: 240 - steps: - - powershell: | - if ("$GITFILESHAREPWD" -ne "" -and "$GITFILESHAREPWD" -ne "`$`(gitfileshare.pwd)") { - net use s: \\gitfileshare.file.core.windows.net\test-cache "$GITFILESHAREPWD" /user:AZURE\gitfileshare /persistent:no - cmd /c mklink /d "$(Build.SourcesDirectory)\test-cache" S:\ - } - displayName: 'Mount test-cache' - env: - GITFILESHAREPWD: $(gitfileshare.pwd) - - powershell: | - $urlbase = "https://dev.azure.com/git-for-windows/git/_apis/build/builds" - $id = ((Invoke-WebRequest -UseBasicParsing "${urlbase}?definitions=22&statusFilter=completed&resultFilter=succeeded&`$top=1").content | ConvertFrom-JSON).value[0].id - $downloadUrl = ((Invoke-WebRequest -UseBasicParsing "${urlbase}/$id/artifacts").content | ConvertFrom-JSON).value[1].resource.downloadUrl - (New-Object Net.WebClient).DownloadFile($downloadUrl,"git-sdk-64-minimal.zip") - Expand-Archive git-sdk-64-minimal.zip -DestinationPath . -Force - Remove-Item git-sdk-64-minimal.zip - - # Let Git ignore the SDK and the test-cache - "/git-sdk-64-minimal/`n/test-cache/`n" | Out-File -NoNewLine -Encoding ascii -Append "$(Build.SourcesDirectory)\.git\info\exclude" - displayName: 'Download git-sdk-64-minimal' - - powershell: | - & git-sdk-64-minimal\usr\bin\bash.exe -lc @" - ci/make-test-artifacts.sh artifacts - "@ - if (!$?) { exit(1) } - displayName: Build - env: - HOME: $(Build.SourcesDirectory) - MSYSTEM: MINGW64 - DEVELOPER: 1 - NO_PERL: 1 - - task: PublishPipelineArtifact@0 - displayName: 'Publish Pipeline Artifact: test artifacts' - inputs: - artifactName: 'windows-artifacts' - targetPath: '$(Build.SourcesDirectory)\artifacts' - - task: PublishPipelineArtifact@0 - displayName: 'Publish Pipeline Artifact: git-sdk-64-minimal' - inputs: - artifactName: 'git-sdk-64-minimal' - targetPath: '$(Build.SourcesDirectory)\git-sdk-64-minimal' - - powershell: | - if ("$GITFILESHAREPWD" -ne "" -and "$GITFILESHAREPWD" -ne "`$`(gitfileshare.pwd)") { - cmd /c rmdir "$(Build.SourcesDirectory)\test-cache" - } - displayName: 'Unmount test-cache' - condition: true - env: - GITFILESHAREPWD: $(gitfileshare.pwd) - -- job: windows_test - displayName: Windows Test - dependsOn: windows_build - condition: succeeded() - pool: - vmImage: windows-latest - timeoutInMinutes: 240 - strategy: - parallel: 10 - steps: - - powershell: | - if ("$GITFILESHAREPWD" -ne "" -and "$GITFILESHAREPWD" -ne "`$`(gitfileshare.pwd)") { - net use s: \\gitfileshare.file.core.windows.net\test-cache "$GITFILESHAREPWD" /user:AZURE\gitfileshare /persistent:no - cmd /c mklink /d "$(Build.SourcesDirectory)\test-cache" S:\ - } - displayName: 'Mount test-cache' - env: - GITFILESHAREPWD: $(gitfileshare.pwd) - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact: test artifacts' - inputs: - artifactName: 'windows-artifacts' - targetPath: '$(Build.SourcesDirectory)' - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact: git-sdk-64-minimal' - inputs: - artifactName: 'git-sdk-64-minimal' - targetPath: '$(Build.SourcesDirectory)\git-sdk-64-minimal' - - powershell: | - & git-sdk-64-minimal\usr\bin\bash.exe -lc @" - test -f artifacts.tar.gz || { - echo No test artifacts found\; skipping >&2 - exit 0 - } - tar xf artifacts.tar.gz || exit 1 - - # Let Git ignore the SDK and the test-cache - printf '%s\n' /git-sdk-64-minimal/ /test-cache/ >>.git/info/exclude - - ci/run-test-slice.sh `$SYSTEM_JOBPOSITIONINPHASE `$SYSTEM_TOTALJOBSINPHASE || { - ci/print-test-failures.sh - exit 1 - } - "@ - if (!$?) { exit(1) } - displayName: 'Test (parallel)' - env: - HOME: $(Build.SourcesDirectory) - MSYSTEM: MINGW64 - NO_SVN_TESTS: 1 - GIT_TEST_SKIP_REBASE_P: 1 - - powershell: | - if ("$GITFILESHAREPWD" -ne "" -and "$GITFILESHAREPWD" -ne "`$`(gitfileshare.pwd)") { - cmd /c rmdir "$(Build.SourcesDirectory)\test-cache" - } - displayName: 'Unmount test-cache' - condition: true - env: - GITFILESHAREPWD: $(gitfileshare.pwd) - - task: PublishTestResults@2 - displayName: 'Publish Test Results **/TEST-*.xml' - inputs: - mergeTestResults: true - testRunTitle: 'windows' - platform: Windows - publishRunAttachments: false - condition: succeededOrFailed() - - task: PublishBuildArtifacts@1 - displayName: 'Publish trash directories of failed tests' - condition: failed() - inputs: - PathtoPublish: t/failed-test-artifacts - ArtifactName: failed-test-artifacts - -- job: vs_build - displayName: Visual Studio Build - condition: succeeded() - pool: - vmImage: windows-latest - timeoutInMinutes: 240 - steps: - - powershell: | - if ("$GITFILESHAREPWD" -ne "" -and "$GITFILESHAREPWD" -ne "`$`(gitfileshare.pwd)") { - net use s: \\gitfileshare.file.core.windows.net\test-cache "$GITFILESHAREPWD" /user:AZURE\gitfileshare /persistent:no - cmd /c mklink /d "$(Build.SourcesDirectory)\test-cache" S:\ - } - displayName: 'Mount test-cache' - env: - GITFILESHAREPWD: $(gitfileshare.pwd) - - powershell: | - $urlbase = "https://dev.azure.com/git-for-windows/git/_apis/build/builds" - $id = ((Invoke-WebRequest -UseBasicParsing "${urlbase}?definitions=22&statusFilter=completed&resultFilter=succeeded&`$top=1").content | ConvertFrom-JSON).value[0].id - $downloadUrl = ((Invoke-WebRequest -UseBasicParsing "${urlbase}/$id/artifacts").content | ConvertFrom-JSON).value[1].resource.downloadUrl - (New-Object Net.WebClient).DownloadFile($downloadUrl,"git-sdk-64-minimal.zip") - Expand-Archive git-sdk-64-minimal.zip -DestinationPath . -Force - Remove-Item git-sdk-64-minimal.zip - - # Let Git ignore the SDK and the test-cache - "/git-sdk-64-minimal/`n/test-cache/`n" | Out-File -NoNewLine -Encoding ascii -Append "$(Build.SourcesDirectory)\.git\info\exclude" - displayName: 'Download git-sdk-64-minimal' - - powershell: | - & git-sdk-64-minimal\usr\bin\bash.exe -lc @" - make NDEBUG=1 DEVELOPER=1 vcxproj - "@ - if (!$?) { exit(1) } - displayName: Generate Visual Studio Solution - env: - HOME: $(Build.SourcesDirectory) - MSYSTEM: MINGW64 - DEVELOPER: 1 - NO_PERL: 1 - GIT_CONFIG_PARAMETERS: "'user.name=CI' 'user.email=ci@git'" - - powershell: | - $urlbase = "https://dev.azure.com/git/git/_apis/build/builds" - $id = ((Invoke-WebRequest -UseBasicParsing "${urlbase}?definitions=9&statusFilter=completed&resultFilter=succeeded&`$top=1").content | ConvertFrom-JSON).value[0].id - $downloadUrl = ((Invoke-WebRequest -UseBasicParsing "${urlbase}/$id/artifacts").content | ConvertFrom-JSON).value[0].resource.downloadUrl - (New-Object Net.WebClient).DownloadFile($downloadUrl, "compat.zip") - Expand-Archive compat.zip -DestinationPath . -Force - Remove-Item compat.zip - displayName: 'Download vcpkg artifacts' - - task: MSBuild@1 - inputs: - solution: git.sln - platform: x64 - configuration: Release - maximumCpuCount: 4 - msbuildArguments: /p:PlatformToolset=v142 - - powershell: | - & compat\vcbuild\vcpkg_copy_dlls.bat release - if (!$?) { exit(1) } - & git-sdk-64-minimal\usr\bin\bash.exe -lc @" - mkdir -p artifacts && - eval \"`$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts | grep ^tar)\" - "@ - if (!$?) { exit(1) } - displayName: Bundle artifact tar - env: - HOME: $(Build.SourcesDirectory) - MSYSTEM: MINGW64 - DEVELOPER: 1 - NO_PERL: 1 - MSVC: 1 - VCPKG_ROOT: $(Build.SourcesDirectory)\compat\vcbuild\vcpkg - - powershell: | - $tag = (Invoke-WebRequest -UseBasicParsing "https://gitforwindows.org/latest-tag.txt").content - $version = (Invoke-WebRequest -UseBasicParsing "https://gitforwindows.org/latest-version.txt").content - $url = "https://github.com/git-for-windows/git/releases/download/${tag}/PortableGit-${version}-64-bit.7z.exe" - (New-Object Net.WebClient).DownloadFile($url,"PortableGit.exe") - & .\PortableGit.exe -y -oartifacts\PortableGit - # Wait until it is unpacked - while (-not @(Remove-Item -ErrorAction SilentlyContinue PortableGit.exe; $?)) { sleep 1 } - displayName: Download & extract portable Git - - task: PublishPipelineArtifact@0 - displayName: 'Publish Pipeline Artifact: MSVC test artifacts' - inputs: - artifactName: 'vs-artifacts' - targetPath: '$(Build.SourcesDirectory)\artifacts' - - powershell: | - if ("$GITFILESHAREPWD" -ne "" -and "$GITFILESHAREPWD" -ne "`$`(gitfileshare.pwd)") { - cmd /c rmdir "$(Build.SourcesDirectory)\test-cache" - } - displayName: 'Unmount test-cache' - condition: true - env: - GITFILESHAREPWD: $(gitfileshare.pwd) - -- job: vs_test - displayName: Visual Studio Test - dependsOn: vs_build - condition: succeeded() - pool: - vmImage: windows-latest - timeoutInMinutes: 240 - strategy: - parallel: 10 - steps: - - powershell: | - if ("$GITFILESHAREPWD" -ne "" -and "$GITFILESHAREPWD" -ne "`$`(gitfileshare.pwd)") { - net use s: \\gitfileshare.file.core.windows.net\test-cache "$GITFILESHAREPWD" /user:AZURE\gitfileshare /persistent:no - cmd /c mklink /d "$(Build.SourcesDirectory)\test-cache" S:\ - } - displayName: 'Mount test-cache' - env: - GITFILESHAREPWD: $(gitfileshare.pwd) - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact: VS test artifacts' - inputs: - artifactName: 'vs-artifacts' - targetPath: '$(Build.SourcesDirectory)' - - powershell: | - & PortableGit\git-cmd.exe --command=usr\bin\bash.exe -lc @" - test -f artifacts.tar.gz || { - echo No test artifacts found\; skipping >&2 - exit 0 - } - tar xf artifacts.tar.gz || exit 1 - - # Let Git ignore the SDK and the test-cache - printf '%s\n' /PortableGit/ /test-cache/ >>.git/info/exclude - - cd t && - PATH=\"`$PWD/helper:`$PATH\" && - test-tool.exe run-command testsuite --jobs=10 -V -x --write-junit-xml \ - `$(test-tool.exe path-utils slice-tests \ - `$SYSTEM_JOBPOSITIONINPHASE `$SYSTEM_TOTALJOBSINPHASE t[0-9]*.sh) - "@ - if (!$?) { exit(1) } - displayName: 'Test (parallel)' - env: - HOME: $(Build.SourcesDirectory) - MSYSTEM: MINGW64 - NO_SVN_TESTS: 1 - GIT_TEST_SKIP_REBASE_P: 1 - - powershell: | - if ("$GITFILESHAREPWD" -ne "" -and "$GITFILESHAREPWD" -ne "`$`(gitfileshare.pwd)") { - cmd /c rmdir "$(Build.SourcesDirectory)\test-cache" - } - displayName: 'Unmount test-cache' - condition: true - env: - GITFILESHAREPWD: $(gitfileshare.pwd) - - task: PublishTestResults@2 - displayName: 'Publish Test Results **/TEST-*.xml' - inputs: - mergeTestResults: true - testRunTitle: 'vs' - platform: Windows - publishRunAttachments: false - condition: succeededOrFailed() - - task: PublishBuildArtifacts@1 - displayName: 'Publish trash directories of failed tests' - condition: failed() - inputs: - PathtoPublish: t/failed-test-artifacts - ArtifactName: failed-vs-test-artifacts - -- job: linux_clang - displayName: linux-clang - condition: succeeded() - pool: - vmImage: ubuntu-latest - steps: - - bash: | - test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 - - sudo apt-get update && - sudo apt-get -y install git gcc make libssl-dev libcurl4-openssl-dev libexpat-dev tcl tk gettext git-email zlib1g-dev apache2-bin && - - export CC=clang || exit 1 - - ci/install-dependencies.sh || exit 1 - ci/run-build-and-tests.sh || { - ci/print-test-failures.sh - exit 1 - } - - test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || sudo umount "$HOME/test-cache" || exit 1 - displayName: 'ci/run-build-and-tests.sh' - env: - GITFILESHAREPWD: $(gitfileshare.pwd) - - task: PublishTestResults@2 - displayName: 'Publish Test Results **/TEST-*.xml' - inputs: - mergeTestResults: true - testRunTitle: 'linux-clang' - platform: Linux - publishRunAttachments: false - condition: succeededOrFailed() - - task: PublishBuildArtifacts@1 - displayName: 'Publish trash directories of failed tests' - condition: failed() - inputs: - PathtoPublish: t/failed-test-artifacts - ArtifactName: failed-test-artifacts - -- job: linux_gcc - displayName: linux-gcc - condition: succeeded() - pool: - vmImage: ubuntu-latest - steps: - - bash: | - test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 - - sudo add-apt-repository ppa:ubuntu-toolchain-r/test && - sudo apt-get update && - sudo apt-get -y install git gcc make libssl-dev libcurl4-openssl-dev libexpat-dev tcl tk gettext git-email zlib1g-dev apache2 language-pack-is git-svn gcc-8 || exit 1 - - ci/install-dependencies.sh || exit 1 - ci/run-build-and-tests.sh || { - ci/print-test-failures.sh - exit 1 - } - - test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || sudo umount "$HOME/test-cache" || exit 1 - displayName: 'ci/run-build-and-tests.sh' - env: - GITFILESHAREPWD: $(gitfileshare.pwd) - - task: PublishTestResults@2 - displayName: 'Publish Test Results **/TEST-*.xml' - inputs: - mergeTestResults: true - testRunTitle: 'linux-gcc' - platform: Linux - publishRunAttachments: false - condition: succeededOrFailed() - - task: PublishBuildArtifacts@1 - displayName: 'Publish trash directories of failed tests' - condition: failed() - inputs: - PathtoPublish: t/failed-test-artifacts - ArtifactName: failed-test-artifacts - -- job: osx_clang - displayName: osx-clang - condition: succeeded() - pool: - vmImage: macOS-latest - steps: - - bash: | - test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 - - export CC=clang - - ci/install-dependencies.sh || exit 1 - ci/run-build-and-tests.sh || { - ci/print-test-failures.sh - exit 1 - } - - test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || umount "$HOME/test-cache" || exit 1 - displayName: 'ci/run-build-and-tests.sh' - env: - GITFILESHAREPWD: $(gitfileshare.pwd) - - task: PublishTestResults@2 - displayName: 'Publish Test Results **/TEST-*.xml' - inputs: - mergeTestResults: true - testRunTitle: 'osx-clang' - platform: macOS - publishRunAttachments: false - condition: succeededOrFailed() - - task: PublishBuildArtifacts@1 - displayName: 'Publish trash directories of failed tests' - condition: failed() - inputs: - PathtoPublish: t/failed-test-artifacts - ArtifactName: failed-test-artifacts - -- job: osx_gcc - displayName: osx-gcc - condition: succeeded() - pool: - vmImage: macOS-latest - steps: - - bash: | - test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 - - ci/install-dependencies.sh || exit 1 - ci/run-build-and-tests.sh || { - ci/print-test-failures.sh - exit 1 - } - - test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || umount "$HOME/test-cache" || exit 1 - displayName: 'ci/run-build-and-tests.sh' - env: - GITFILESHAREPWD: $(gitfileshare.pwd) - - task: PublishTestResults@2 - displayName: 'Publish Test Results **/TEST-*.xml' - inputs: - mergeTestResults: true - testRunTitle: 'osx-gcc' - platform: macOS - publishRunAttachments: false - condition: succeededOrFailed() - - task: PublishBuildArtifacts@1 - displayName: 'Publish trash directories of failed tests' - condition: failed() - inputs: - PathtoPublish: t/failed-test-artifacts - ArtifactName: failed-test-artifacts - -- job: gettext_poison - displayName: GETTEXT_POISON - condition: succeeded() - pool: - vmImage: ubuntu-latest - steps: - - bash: | - test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 - - sudo apt-get update && - sudo apt-get -y install git gcc make libssl-dev libcurl4-openssl-dev libexpat-dev tcl tk gettext git-email zlib1g-dev && - - export jobname=GETTEXT_POISON || exit 1 - - ci/run-build-and-tests.sh || { - ci/print-test-failures.sh - exit 1 - } - - test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || sudo umount "$HOME/test-cache" || exit 1 - displayName: 'ci/run-build-and-tests.sh' - env: - GITFILESHAREPWD: $(gitfileshare.pwd) - - task: PublishTestResults@2 - displayName: 'Publish Test Results **/TEST-*.xml' - inputs: - mergeTestResults: true - testRunTitle: 'gettext-poison' - platform: Linux - publishRunAttachments: false - condition: succeededOrFailed() - - task: PublishBuildArtifacts@1 - displayName: 'Publish trash directories of failed tests' - condition: failed() - inputs: - PathtoPublish: t/failed-test-artifacts - ArtifactName: failed-test-artifacts - -- job: linux32 - displayName: Linux32 - condition: succeeded() - pool: - vmImage: ubuntu-latest - steps: - - bash: | - test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 - - res=0 - sudo AGENT_OS="$AGENT_OS" BUILD_BUILDNUMBER="$BUILD_BUILDNUMBER" BUILD_REPOSITORY_URI="$BUILD_REPOSITORY_URI" BUILD_SOURCEBRANCH="$BUILD_SOURCEBRANCH" BUILD_SOURCEVERSION="$BUILD_SOURCEVERSION" SYSTEM_PHASENAME="$SYSTEM_PHASENAME" SYSTEM_TASKDEFINITIONSURI="$SYSTEM_TASKDEFINITIONSURI" SYSTEM_TEAMPROJECT="$SYSTEM_TEAMPROJECT" CC=$CC MAKEFLAGS="$MAKEFLAGS" jobname=Linux32 bash -lxc ci/run-docker.sh || res=1 - - sudo chmod a+r t/out/TEST-*.xml - test ! -d t/failed-test-artifacts || sudo chmod a+r t/failed-test-artifacts - - test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || sudo umount "$HOME/test-cache" || res=1 - exit $res - displayName: 'jobname=Linux32 ci/run-docker.sh' - env: - GITFILESHAREPWD: $(gitfileshare.pwd) - - task: PublishTestResults@2 - displayName: 'Publish Test Results **/TEST-*.xml' - inputs: - mergeTestResults: true - testRunTitle: 'linux32' - platform: Linux - publishRunAttachments: false - condition: succeededOrFailed() - - task: PublishBuildArtifacts@1 - displayName: 'Publish trash directories of failed tests' - condition: failed() - inputs: - PathtoPublish: t/failed-test-artifacts - ArtifactName: failed-test-artifacts - -- job: static_analysis - displayName: StaticAnalysis - condition: succeeded() - pool: - vmImage: ubuntu-latest - steps: - - bash: | - test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 - - sudo apt-get update && - sudo apt-get install -y coccinelle libcurl4-openssl-dev libssl-dev libexpat-dev gettext && - - export jobname=StaticAnalysis && - - ci/run-static-analysis.sh || exit 1 - - test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || sudo umount "$HOME/test-cache" || exit 1 - displayName: 'ci/run-static-analysis.sh' - env: - GITFILESHAREPWD: $(gitfileshare.pwd) - -- job: documentation - displayName: Documentation - condition: succeeded() - pool: - vmImage: ubuntu-latest - steps: - - bash: | - test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || ci/mount-fileshare.sh //gitfileshare.file.core.windows.net/test-cache gitfileshare "$GITFILESHAREPWD" "$HOME/test-cache" || exit 1 - - sudo apt-get update && - sudo apt-get install -y asciidoc xmlto asciidoctor docbook-xsl-ns && - - export ALREADY_HAVE_ASCIIDOCTOR=yes. && - export jobname=Documentation && - - ci/test-documentation.sh || exit 1 - - test "$GITFILESHAREPWD" = '$(gitfileshare.pwd)' || sudo umount "$HOME/test-cache" || exit 1 - displayName: 'ci/test-documentation.sh' - env: - GITFILESHAREPWD: $(gitfileshare.pwd) From bcf50f3eda6415c4bc7729317db1a95120d62146 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 7 Apr 2020 09:31:34 +0700 Subject: [PATCH 25/27] tests: when run in Bash, annotate test failures with file name/line number MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a test fails, it is nice to see where the corresponding code lives in the worktree. Sadly, it seems that only Bash allows us to infer this information. Let's do it when we detect that we're running in a Bash. This will come in handy in the next commit, where we teach the GitHub Actions workflow to annotate failed test runs with this information. Signed-off-by: Johannes Schindelin Signed-off-by: Đoàn Trần Công Danh --- t/test-lib.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/t/test-lib.sh b/t/test-lib.sh index 0ea1e5a05edd86..40a00983f7013e 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -657,6 +657,18 @@ die () { fi } +file_lineno () { + test -z "$GIT_TEST_FRAMEWORK_SELFTEST" && test -n "$BASH" || return 0 + local i + for i in ${!BASH_SOURCE[*]} + do + case $i,"${BASH_SOURCE[$i]##*/}" in + 0,t[0-9]*.sh) echo "t/${BASH_SOURCE[$i]}:$LINENO: ${1+$1: }"; return;; + *,t[0-9]*.sh) echo "t/${BASH_SOURCE[$i]}:${BASH_LINENO[$(($i-1))]}: ${1+$1: }"; return;; + esac + done +} + GIT_EXIT_OK= trap 'die' EXIT # Disable '-x' tracing, because with some shells, notably dash, it @@ -702,7 +714,7 @@ test_failure_ () { write_junit_xml_testcase "$1" " $junit_insert" fi test_failure=$(($test_failure + 1)) - say_color error "not ok $test_count - $1" + say_color error "$(file_lineno error)not ok $test_count - $1" shift printf '%s\n' "$*" | sed -e 's/^/# /' test "$immediate" = "" || { finalize_junit_xml; GIT_EXIT_OK=t; exit 1; } From 651fd37cd5357b89ee00daedabe8a433ad404256 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 7 Apr 2020 09:31:35 +0700 Subject: [PATCH 26/27] ci: add a problem matcher for GitHub Actions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With this patch, test failures will be annotated with a helpful, clickable message in GitHub Actions. For details, see https://github.com/actions/toolkit/blob/master/docs/problem-matchers.md Note: we need to set `TEST_SHELL_PATH` to Bash so that the problem matcher is fed a file and line number for each test failure. Signed-off-by: Johannes Schindelin Signed-off-by: Đoàn Trần Công Danh --- ci/git-problem-matcher.json | 16 ++++++++++++++++ ci/lib.sh | 5 +++++ 2 files changed, 21 insertions(+) create mode 100644 ci/git-problem-matcher.json diff --git a/ci/git-problem-matcher.json b/ci/git-problem-matcher.json new file mode 100644 index 00000000000000..506dfbd97f0eb3 --- /dev/null +++ b/ci/git-problem-matcher.json @@ -0,0 +1,16 @@ +{ + "problemMatcher": [ + { + "owner": "git-test-suite", + "pattern": [ + { + "regexp": "^([^ :]+\\.sh):(\\d+): (error|warning|info):\\s+(.*)$", + "file": 1, + "line": 2, + "severity": 3, + "message": 4 + } + ] + } + ] +} diff --git a/ci/lib.sh b/ci/lib.sh index 8b39624f3cc462..4c54540fa8961e 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -157,6 +157,11 @@ then MAKEFLAGS="$MAKEFLAGS --jobs=10" test windows != "$CI_OS_NAME" || GIT_TEST_OPTS="--no-chain-lint --no-bin-wrappers $GIT_TEST_OPTS" + + # https://github.com/actions/toolkit/blob/master/docs/commands.md#problem-matchers + echo "::add-matcher::ci/git-problem-matcher.json" + test linux-musl = "$jobname" || + MAKEFLAGS="$MAKEFLAGS TEST_SHELL_PATH=/bin/sh" else echo "Could not identify CI type" >&2 env >&2 From 36a62374a9896c3ddd00be116201cecbec6c38b4 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 7 Apr 2020 09:31:36 +0700 Subject: [PATCH 27/27] ci: let GitHub Actions upload failed tests' directories MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Arguably, CI builds' most important task is to not only identify regressions, but to make it as easy as possible to investigate what went wrong. In that light, we will want to provide users with a way to inspect the tests' output as well as the corresponding directories. This commit adds build steps that are only executed when tests failed, uploading the relevant information as build artifacts. These artifacts can then be downloaded by interested parties to diagnose the failures more efficiently. Signed-off-by: Johannes Schindelin Signed-off-by: Đoàn Trần Công Danh --- .github/workflows/main.yml | 18 ++++++++++++++++++ ci/print-test-failures.sh | 7 +++++++ 2 files changed, 25 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e1ac6d23b4c467..fd4df939b50e48 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -63,6 +63,12 @@ jobs: shell: powershell run: | & .\git-sdk-64-minimal\usr\bin\bash.exe -lc ci/print-test-failures.sh + - name: Upload failed tests' directories + if: failure() && env.FAILED_TEST_ARTIFACTS != '' + uses: actions/upload-artifact@v1 + with: + name: failed-tests-windows + path: ${{env.FAILED_TEST_ARTIFACTS}} vs-build: env: MSYSTEM: MINGW64 @@ -176,6 +182,12 @@ jobs: - run: ci/run-build-and-tests.sh - run: ci/print-test-failures.sh if: failure() + - name: Upload failed tests' directories + if: failure() && env.FAILED_TEST_ARTIFACTS != '' + uses: actions/upload-artifact@v1 + with: + name: failed-tests-${{matrix.vector.jobname}} + path: ${{env.FAILED_TEST_ARTIFACTS}} dockerized: strategy: matrix: @@ -194,6 +206,12 @@ jobs: - run: ci/run-build-and-tests.sh - run: ci/print-test-failures.sh if: failure() + - name: Upload failed tests' directories + if: failure() && env.FAILED_TEST_ARTIFACTS != '' + uses: actions/upload-artifact@v1 + with: + name: failed-tests-${{matrix.vector.jobname}} + path: ${{env.FAILED_TEST_ARTIFACTS}} static-analysis: env: jobname: StaticAnalysis diff --git a/ci/print-test-failures.sh b/ci/print-test-failures.sh index e688a26f0d6146..92a983a265c246 100755 --- a/ci/print-test-failures.sh +++ b/ci/print-test-failures.sh @@ -46,6 +46,13 @@ do mv "$trash_dir" failed-test-artifacts continue ;; + github-actions) + mkdir -p failed-test-artifacts + echo "::set-env name=FAILED_TEST_ARTIFACTS::t/failed-test-artifacts" + cp "${TEST_EXIT%.exit}.out" failed-test-artifacts/ + tar czf failed-test-artifacts/"$test_name".trash.tar.gz "$trash_dir" + continue + ;; *) echo "Unhandled CI type: $CI_TYPE" >&2 exit 1