Permalink
Browse files

Merge branch 'ab/test-2'

* ab/test-2: (51 commits)
  tests: factor HOME=$(pwd) in test-lib.sh
  test-lib: use subshell instead of cd $new && .. && cd $old
  tests: simplify "missing PREREQ" message
  t/t0000-basic.sh: Run the passing TODO test inside its own test-lib
  test-lib: Allow overriding of TEST_DIRECTORY
  test-lib: Use "$GIT_BUILD_DIR" instead of "$TEST_DIRECTORY"/../
  test-lib: Use $TEST_DIRECTORY or $GIT_BUILD_DIR instead of $(pwd) and ../
  test: Introduce $GIT_BUILD_DIR
  cvs tests: do not touch test CVS repositories shipped with source
  t/t9602-cvsimport-branches-tags.sh: Add a PERL prerequisite
  t/t9601-cvsimport-vendor-branch.sh: Add a PERL prerequisite
  t/t7105-reset-patch.sh: Add a PERL prerequisite
  t/t9001-send-email.sh: convert setup code to tests
  t/t9001-send-email.sh: change from skip_all=* to prereq skip
  t/t9001-send-email.sh: Remove needless PROG=* assignment
  t/t9600-cvsimport.sh: change from skip_all=* to prereq skip
  lib-patch-mode tests: change from skip_all=* to prereq skip
  t/t3701-add-interactive.sh: change from skip_all=* to prereq skip
  tests: Move FILEMODE prerequisite to lib-prereq-FILEMODE.sh
  t/Makefile: Create test-results dir for smoke target
  ...

Conflicts:
	t/t6035-merge-dir-to-symlink.sh
  • Loading branch information...
2 parents b480d38 + e4c62e6 commit a2c6726417db22bea7a878aeb88e18bcc95e5c0c @gitster gitster committed Sep 4, 2010
Showing with 968 additions and 559 deletions.
  1. +6 −0 .gitignore
  2. +22 −3 Makefile
  3. +40 −2 t/Makefile
  4. +193 −8 t/README
  5. +21 −0 t/harness
  6. +6 −3 t/lib-cvs.sh
  7. +0 −5 t/lib-patch-mode.sh
  8. +11 −0 t/lib-prereq-FILEMODE.sh
  9. +52 −0 t/t0000-basic.sh
  10. +1 −7 t/t0001-init.sh
  11. +4 −4 t/t0004-unwritable.sh
  12. +1 −1 t/t1004-read-tree-m-u-wf.sh
  13. +8 −6 t/t1304-default-acl.sh
  14. +4 −10 t/t2007-checkout-symlink.sh
  15. +14 −14 t/t2016-checkout-patch.sh
  16. +60 −22 t/t3300-funny-names.sh
  17. +20 −12 t/t3302-notes-index-expensive.sh
  18. +0 −16 t/t3600-rm.sh
  19. +5 −5 t/t3700-add.sh
  20. +71 −41 t/t3701-add-interactive.sh
  21. +21 −17 t/t3902-quoted.sh
  22. +5 −5 t/t3904-stash-patch.sh
  23. +7 −11 t/t4004-diff-rename-symlink.sh
  24. +6 −12 t/t4011-diff-symlink.sh
  25. +15 −7 t/t4016-diff-quote.sh
  26. +4 −10 t/t4023-diff-rename-typechange.sh
  27. +1 −7 t/t4102-apply-rename.sh
  28. +11 −17 t/t4114-apply-typechange.sh
  29. +3 −9 t/t4115-apply-symlink.sh
  30. +3 −9 t/t4122-apply-symlink-inside.sh
  31. +1 −7 t/t4129-apply-samemode.sh
  32. +24 −9 t/t5503-tagfollow.sh
  33. +4 −10 t/t5522-pull-symlink.sh
  34. +0 −2 t/t5601-clone.sh
  35. +9 −7 t/t5705-clone-2gb.sh
  36. +11 −13 t/t5800-remote-helpers.sh
  37. +1 −5 t/t6031-merge-recursive.sh
  38. +16 −22 t/t6035-merge-dir-to-symlink.sh
  39. +5 −5 t/t7005-editor.sh
  40. +8 −8 t/t7105-reset-patch.sh
  41. +3 −4 t/t7300-clean.sh
  42. +1 −1 t/t7508-status.sh
  43. +22 −27 t/t7800-difftool.sh
  44. +123 −86 t/t9001-send-email.sh
  45. +0 −2 t/t9130-git-svn-authors-file.sh
  46. +1 −5 t/t9200-git-cvsexportcommit.sh
  47. +18 −21 t/t9600-cvsimport.sh
  48. +10 −11 t/t9601-cvsimport-vendor-branch.sh
  49. +12 −13 t/t9602-cvsimport-branches-tags.sh
  50. +1 −2 t/t9603-cvsimport-patchsets.sh
  51. +83 −36 t/test-lib.sh
View
@@ -186,6 +186,12 @@
*.[aos]
*.py[co]
.depend/
+*.gcda
+*.gcno
+*.gcov
+/coverage-untested-functions
+/cover_db/
+/cover_db_html/
*+
/config.mak
/autom4te.cache
View
@@ -310,6 +310,7 @@ TCL_PATH = tclsh
TCLTK_PATH = wish
PTHREAD_LIBS = -lpthread
PTHREAD_CFLAGS =
+GCOV = gcov
export TCL_PATH TCLTK_PATH
@@ -1499,6 +1500,7 @@ ifndef V
QUIET_BUILT_IN = @echo ' ' BUILTIN $@;
QUIET_GEN = @echo ' ' GEN $@;
QUIET_LNCP = @echo ' ' LN/CP $@;
+ QUIET_GCOV = @echo ' ' GCOV $@;
QUIET_SUBDIR0 = +@subdir=
QUIET_SUBDIR1 = ;$(NO_SUBDIR) echo ' ' SUBDIR $$subdir; \
$(MAKE) $(PRINT_DIR) -C $$subdir
@@ -2324,19 +2326,36 @@ coverage:
$(MAKE) coverage-build
$(MAKE) coverage-report
+object_dirs := $(sort $(dir $(OBJECTS)))
coverage-clean:
- rm -f *.gcda *.gcno
+ $(RM) $(addsuffix *.gcov,$(object_dirs))
+ $(RM) $(addsuffix *.gcda,$(object_dirs))
+ $(RM) $(addsuffix *.gcno,$(object_dirs))
+ $(RM) coverage-untested-functions
+ $(RM) -r cover_db/
+ $(RM) -r cover_db_html/
COVERAGE_CFLAGS = $(CFLAGS) -O0 -ftest-coverage -fprofile-arcs
COVERAGE_LDFLAGS = $(CFLAGS) -O0 -lgcov
+GCOVFLAGS = --preserve-paths --branch-probabilities --all-blocks
coverage-build: coverage-clean
$(MAKE) CFLAGS="$(COVERAGE_CFLAGS)" LDFLAGS="$(COVERAGE_LDFLAGS)" all
$(MAKE) CFLAGS="$(COVERAGE_CFLAGS)" LDFLAGS="$(COVERAGE_LDFLAGS)" \
-j1 test
coverage-report:
- gcov -b *.c
+ $(QUIET_GCOV)for dir in $(object_dirs); do \
+ $(GCOV) $(GCOVFLAGS) --object-directory=$$dir $$dir*.c || exit; \
+ done
+
+coverage-untested-functions: coverage-report
grep '^function.*called 0 ' *.c.gcov \
| sed -e 's/\([^:]*\)\.gcov: *function \([^ ]*\) called.*/\1: \2/' \
- | tee coverage-untested-functions
+ > coverage-untested-functions
+
+cover_db: coverage-report
+ gcov2perl -db cover_db *.gcov
+
+cover_db_html: cover_db
+ cover -report html -outputdir cover_db_html cover_db
View
@@ -8,6 +8,7 @@
#GIT_TEST_OPTS=--verbose --debug
SHELL_PATH ?= $(SHELL)
+PERL_PATH ?= /usr/bin/perl
TAR ?= $(TAR)
RM ?= rm -f
@@ -28,7 +29,6 @@ pre-clean:
clean:
$(RM) -r 'trash directory'.* test-results
- $(RM) t????/cvsroot/CVSROOT/?*
$(RM) -r valgrind/bin
$(RM) .prove
@@ -49,4 +49,42 @@ full-svn-test:
valgrind:
GIT_TEST_OPTS=--valgrind $(MAKE)
-.PHONY: pre-clean $(T) aggregate-results clean valgrind
+# Smoke testing targets
+-include ../GIT-VERSION-FILE
+uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo unknown')
+uname_M := $(shell sh -c 'uname -m 2>/dev/null || echo unknown')
+
+test-results:
+ mkdir -p test-results
+
+test-results/git-smoke.tar.gz: test-results
+ $(PERL_PATH) ./harness \
+ --archive="test-results/git-smoke.tar.gz" \
+ $(T)
+
+smoke: test-results/git-smoke.tar.gz
+
+SMOKE_UPLOAD_FLAGS =
+ifdef SMOKE_USERNAME
+ SMOKE_UPLOAD_FLAGS += -F username="$(SMOKE_USERNAME)" -F password="$(SMOKE_PASSWORD)"
+endif
+ifdef SMOKE_COMMENT
+ SMOKE_UPLOAD_FLAGS += -F comments="$(SMOKE_COMMENT)"
+endif
+ifdef SMOKE_TAGS
+ SMOKE_UPLOAD_FLAGS += -F tags="$(SMOKE_TAGS)"
+endif
+
+smoke_report: smoke
+ curl \
+ -H "Expect: " \
+ -F project=Git \
+ -F architecture="$(uname_M)" \
+ -F platform="$(uname_S)" \
+ -F revision="$(GIT_VERSION)" \
+ -F report_file=@test-results/git-smoke.tar.gz \
+ $(SMOKE_UPLOAD_FLAGS) \
+ http://smoke.git.nix.is/app/projects/process_add_report/1 \
+ | grep -v ^Redirecting
+
+.PHONY: pre-clean $(T) aggregate-results clean valgrind smoke smoke_report
View
201 t/README
@@ -268,6 +268,18 @@ Do:
git push gh &&
test ...
+ - Check the test coverage for your tests. See the "Test coverage"
+ below.
+
+ Don't blindly follow test coverage metrics, they're a good way to
+ spot if you've missed something. If a new function you added
+ doesn't have any coverage you're probably doing something wrong,
+ but having 100% coverage doesn't necessarily mean that you tested
+ everything.
+
+ Tests that are likely to smoke out future regressions are better
+ than tests that just inflate the coverage metrics.
+
Don't:
- exit() within a <script> part.
@@ -307,16 +319,31 @@ Keep in mind:
Skipping tests
--------------
-If you need to skip all the remaining tests you should set skip_all
-and immediately call test_done. The string you give to skip_all will
-be used as an explanation for why the test was skipped. for instance:
+If you need to skip tests you should do so be using the three-arg form
+of the test_* functions (see the "Test harness library" section
+below), e.g.:
+
+ test_expect_success PERL 'I need Perl' "
+ '$PERL_PATH' -e 'hlagh() if unf_unf()'
+ "
+
+The advantage of skipping tests like this is that platforms that don't
+have the PERL and other optional dependencies get an indication of how
+many tests they're missing.
+
+If the test code is too hairy for that (i.e. does a lot of setup work
+outside test assertions) you can also skip all remaining tests by
+setting skip_all and immediately call test_done:
if ! test_have_prereq PERL
then
skip_all='skipping perl interface tests, perl not available'
test_done
fi
+The string you give to skip_all will be used as an explanation for why
+the test was skipped.
+
End with test_done
------------------
@@ -350,6 +377,12 @@ library for your script to use.
test_expect_success TTY 'git --paginate rev-list uses a pager' \
' ... '
+ You can also supply a comma-separated list of prerequisites, in the
+ rare case where your test depends on more than one:
+
+ test_expect_success PERL,PYTHON 'yo dawg' \
+ ' test $(perl -E 'print eval "1 +" . qx[python -c "print 2"]') == "4" '
+
- test_expect_failure [<prereq>] <message> <script>
This is NOT the opposite of test_expect_success, but is used
@@ -404,11 +437,12 @@ library for your script to use.
- test_set_prereq SOME_PREREQ
Set a test prerequisite to be used later with test_have_prereq. The
- test-lib will set some prerequisites for you, e.g. PERL and PYTHON
- which are derived from ./GIT-BUILD-OPTIONS (grep test_set_prereq
- test-lib.sh for more). Others you can set yourself and use later
- with either test_have_prereq directly, or the three argument
- invocation of test_expect_success and test_expect_failure.
+ test-lib will set some prerequisites for you, see the
+ "Prerequisites" section below for a full list of these.
+
+ Others you can set yourself and use later with either
+ test_have_prereq directly, or the three argument invocation of
+ test_expect_success and test_expect_failure.
- test_have_prereq SOME PREREQ
@@ -488,6 +522,45 @@ library for your script to use.
...
'
+Prerequisites
+-------------
+
+These are the prerequisites that the test library predefines with
+test_have_prereq.
+
+See the prereq argument to the test_* functions in the "Test harness
+library" section above and the "test_have_prereq" function for how to
+use these, and "test_set_prereq" for how to define your own.
+
+ - PERL & PYTHON
+
+ Git wasn't compiled with NO_PERL=YesPlease or
+ NO_PYTHON=YesPlease. Wrap any tests that need Perl or Python in
+ these.
+
+ - POSIXPERM
+
+ The filesystem supports POSIX style permission bits.
+
+ - BSLASHPSPEC
+
+ Backslashes in pathspec are not directory separators. This is not
+ set on Windows. See 6fd1106a for details.
+
+ - EXECKEEPSPID
+
+ The process retains the same pid across exec(2). See fb9a2bea for
+ details.
+
+ - SYMLINKS
+
+ The filesystem we're on supports symbolic links. E.g. a FAT
+ filesystem doesn't support these. See 704a3143 for details.
+
+ - SANITY
+
+ Test is not run by root user, and an attempt to write to an
+ unwritable file is expected to fail correctly.
Tips for Writing Tests
----------------------
@@ -515,3 +588,115 @@ the purpose of t0000-basic.sh, which is to isolate that level of
validation in one place. Your test also ends up needing
updating when such a change to the internal happens, so do _not_
do it and leave the low level of validation to t0000-basic.sh.
+
+Test coverage
+-------------
+
+You can use the coverage tests to find code paths that are not being
+used or properly exercised yet.
+
+To do that, run the coverage target at the top-level (not in the t/
+directory):
+
+ make coverage
+
+That'll compile Git with GCC's coverage arguments, and generate a test
+report with gcov after the tests finish. Running the coverage tests
+can take a while, since running the tests in parallel is incompatible
+with GCC's coverage mode.
+
+After the tests have run you can generate a list of untested
+functions:
+
+ make coverage-untested-functions
+
+You can also generate a detailed per-file HTML report using the
+Devel::Cover module. To install it do:
+
+ # On Debian or Ubuntu:
+ sudo aptitude install libdevel-cover-perl
+
+ # From the CPAN with cpanminus
+ curl -L http://cpanmin.us | perl - --sudo --self-upgrade
+ cpanm --sudo Devel::Cover
+
+Then, at the top-level:
+
+ make cover_db_html
+
+That'll generate a detailed cover report in the "cover_db_html"
+directory, which you can then copy to a webserver, or inspect locally
+in a browser.
+
+Smoke testing
+-------------
+
+The Git test suite has support for smoke testing. Smoke testing is
+when you submit the results of a test run to a central server for
+analysis and aggregation.
+
+Running a smoke tester is an easy and valuable way of contributing to
+Git development, particularly if you have access to an uncommon OS on
+obscure hardware.
+
+After building Git you can generate a smoke report like this in the
+"t" directory:
+
+ make clean smoke
+
+You can also pass arguments via the environment. This should make it
+faster:
+
+ GIT_TEST_OPTS='--root=/dev/shm' TEST_JOBS=10 make clean smoke
+
+The "smoke" target will run the Git test suite with Perl's
+"TAP::Harness" module, and package up the results in a .tar.gz archive
+with "TAP::Harness::Archive". The former is included with Perl v5.10.1
+or later, but you'll need to install the latter from the CPAN. See the
+"Test coverage" section above for how you might do that.
+
+Once the "smoke" target finishes you'll see a message like this:
+
+ TAP Archive created at <path to git>/t/test-results/git-smoke.tar.gz
+
+To upload the smoke report you need to have curl(1) installed, then
+do:
+
+ make smoke_report
+
+To upload the report anonymously. Hopefully that'll return something
+like "Reported #7 added.".
+
+If you're going to be uploading reports frequently please request a
+user account by E-Mailing gitsmoke@v.nix.is. Once you have a username
+and password you'll be able to do:
+
+ SMOKE_USERNAME=<username> SMOKE_PASSWORD=<password> make smoke_report
+
+You can also add an additional comment to attach to the report, and/or
+a comma separated list of tags:
+
+ SMOKE_USERNAME=<username> SMOKE_PASSWORD=<password> \
+ SMOKE_COMMENT=<comment> SMOKE_TAGS=<tags> \
+ make smoke_report
+
+Once the report is uploaded it'll be made available at
+http://smoke.git.nix.is, here's an overview of Recent Smoke Reports
+for Git:
+
+ http://smoke.git.nix.is/app/projects/smoke_reports/1
+
+The reports will also be mirrored to GitHub every few hours:
+
+ http://github.com/gitsmoke/smoke-reports
+
+The Smolder SQLite database is also mirrored and made available for
+download:
+
+ http://github.com/gitsmoke/smoke-database
+
+Note that the database includes hashed (with crypt()) user passwords
+and E-Mail addresses. Don't use a valuable password for the smoke
+service if you have an account, or an E-Mail address you don't want to
+be publicly known. The user accounts are just meant to be convenient
+labels, they're not meant to be secure.
View
@@ -0,0 +1,21 @@
+#!/usr/bin/perl
+use strict;
+use warnings;
+use Getopt::Long ();
+use TAP::Harness::Archive;
+
+Getopt::Long::Parser->new(
+ config => [ qw/ pass_through / ],
+)->getoptions(
+ 'jobs:1' => \(my $jobs = $ENV{TEST_JOBS}),
+ 'archive=s' => \my $archive,
+) or die "$0: Couldn't getoptions()";
+
+TAP::Harness::Archive->new({
+ jobs => $jobs,
+ archive => $archive,
+ ($ENV{GIT_TEST_OPTS}
+ ? (test_args => [ split /\s+/, $ENV{GIT_TEST_OPTS} ])
+ : ()),
+ extra_properties => {},
+})->runtests(@ARGV);
Oops, something went wrong.

0 comments on commit a2c6726

Please sign in to comment.