Merge pull request #28 from zimmski/19-create-simpler-simple.sh

Create an exec script to test only the package under test

Makefile

@@ -22,7 +22,7 @@ clean-coverage:
 	find $(ROOT_DIR) | grep .coverprofile | xargs rm
 generate: clean
 	go generate $(PKG)/...
-install: generate
+install:
 	go install -v $(PKG)/...
 install-dependencies:
 	go get -t -v $(PKG)/...

README.md

@@ -8,7 +8,7 @@ The following command mutates the go-mutesting project with all available mutato
 
 ```bash
 cd $GOPATH/src/github.com/zimmski/go-mutesting
-go-mutesting --exec "$GOPATH/src/github.com/zimmski/go-mutesting/scripts/simple.sh" --exec-timeout 1 github.com/zimmski/go-mutesting/...
+go-mutesting --exec "$GOPATH/src/github.com/zimmski/go-mutesting/scripts/test-mutated-package.sh" --exec-timeout 1 github.com/zimmski/go-mutesting/...
 ```
 
 The execution of this command prints for every mutation if it was successfully tested or not. If not, the source code patch is printed out, so the mutation can be investigated. The following shows an example for a patch of a mutation.
@@ -79,19 +79,19 @@ go-mutesting --help
 
 > **Note**: This README describes only a few of the available options and arguments. It is therefore advisable to examine the help.
 
-The targets of the mutation testing can be defined as arguments to the binary. Every target can be either a Go source file, a directory or a package. Directories and packages can also include the `...`wildcard pattern which will search recursively for Go source files. Test source files with the ending `_test` are excluded, since this would interfere with testing the mutation most of the time.
+The targets of the mutation testing can be defined as arguments to the binary. Every target can be either a Go source file, a directory or a package. Directories and packages can also include the `...` wildcard pattern which will search recursively for Go source files. Test source files with the ending `_test` are excluded, since this would interfere with testing the mutation most of the time.
 
 The following example will gather all Go files which are defined by the targets and generate mutations with all available mutators of the binary.
 
 ```bash
 go-mutesting parse.go example/ github.com/zimmski/go-mutesting/mutator/...
 ```
 
-Since every mutation has to be tested it is necessary to define a [command](#write-mutation-exec-commands) with the `--exec` option. The [scripts](/scripts) directory holds basic exec commands for Go projects. The [simple.sh](/scripts/simple.sh) script for example implements the replacement of the original file with the mutation, the execution of all tests of the current directory and sub-directories, and the reporting if the mutation was killed. It can be for example used to test the [github.com/zimmski/go-mutesting/example](/example) package.
+Since every mutation has to be tested it is necessary to define a [command](#write-mutation-exec-commands) with the `--exec` option. The [scripts](/scripts) directory holds basic exec commands for Go projects. The [test-mutated-package.sh](/scripts/test-mutated-package.sh) script for example implements the replacement of the original file with the mutation, the execution of all tests of the package of the mutated file, and the reporting if the mutation was killed. It can be for example used to test the [github.com/zimmski/go-mutesting/example](/example) package.
 
 ```bash
 cd $GOPATH/src/github.com/zimmski/go-mutesting/example
-go-mutesting --exec "$GOPATH/src/github.com/zimmski/go-mutesting/scripts/simple.sh" github.com/zimmski/go-mutesting/example
+go-mutesting --exec "$GOPATH/src/github.com/zimmski/go-mutesting/scripts/test-mutated-package.sh" github.com/zimmski/go-mutesting/example
 ```
 
 The execution will print the following output.
@@ -151,7 +151,7 @@ The blacklist file, which is named `example.blacklist` in this example, can then
 
 ```bash
 cd $GOPATH/src/github.com/zimmski/go-mutesting/example
-go-mutesting --exec "$GOPATH/src/github.com/zimmski/go-mutesting/scripts/simple.sh" --blacklist example.blacklist github.com/zimmski/go-mutesting/example
+go-mutesting --exec "$GOPATH/src/github.com/zimmski/go-mutesting/scripts/test-mutated-package.sh" --blacklist example.blacklist github.com/zimmski/go-mutesting/example
 ```
 
 The execution will print the following output.
@@ -200,8 +200,10 @@ The command is given a set of environment variables which define exactly one mut
 | MUTATE_CHANGED  | Defines the filename to the mutation of the original file.     |
 | MUTATE_DEBUG    | Defines if debugging output should be printed.                 |
 | MUTATE_ORIGINAL | Defines the filename to the original file which was mutated.   |
+| MUTATE_PACKAGE  | Defines the import path of the origianl file.                  |
 | MUTATE_TIMEOUT  | Defines a timeout which should be honored by the exec command. |
 | MUTATE_VERBOSE  | Defines if verbose output should be printed.                   |
+| TEST_RECURSIVE  | Defines if tests should be run recursively.                    |
 
 A command must exit with an appropriate exit code.
 

cmd/go-mutesting/main.go

@@ -5,6 +5,7 @@ import (
 	"crypto/md5"
 	"fmt"
 	"go/ast"
+	"go/build"
 	"go/format"
 	"go/printer"
 	"go/token"
@@ -70,6 +71,10 @@ type options struct {
 		Timeout uint   `long:"exec-timeout" description:"Sets a timeout for the command execution (in seconds)" default:"10"`
 	} `group:"Exec options"`
 
+	Test struct {
+		Recursive bool `long:"test-recursive" description:"Defines if the executer should test recursively"`
+	} `group:"Test options"`
+
 	Remaining struct {
 		Targets []string `description:"Packages, directories and files even with patterns"`
 	} `positional-args:"true" required:"true"`
@@ -286,6 +291,11 @@ MUTATOR:
 }
 
 func mutate(opts *options, mutators []mutator.Mutator, mutationBlackList map[string]struct{}, mutationID int, file string, fset *token.FileSet, src ast.Node, node ast.Node, tmpFile string, execs []string, stats *Stats) int {
+	p, err := build.ImportDir(filepath.Dir(file), build.FindOnly)
+	if err != nil {
+		panic(err)
+	}
+
 	for _, m := range mutators {
 		debug(opts, "Mutator %s", m)
 
@@ -320,9 +330,13 @@ func mutate(opts *options, mutators []mutator.Mutator, mutationBlackList map[str
 						"MUTATE_CHANGED=" + mutationFile,
 						fmt.Sprintf("MUTATE_DEBUG=%t", opts.General.Debug),
 						"MUTATE_ORIGINAL=" + file,
+						"MUTATE_PACKAGE=" + p.ImportPath,
 						fmt.Sprintf("MUTATE_TIMEOUT=%d", opts.Exec.Timeout),
 						fmt.Sprintf("MUTATE_VERBOSE=%t", opts.General.Verbose),
 					}...)
+					if opts.Test.Recursive {
+						execCommand.Env = append(execCommand.Env, "TEST_RECURSIVE=true")
+					}
 
 					err = execCommand.Start()
 					if err != nil {

cmd/go-mutesting/main_test.go

@@ -13,7 +13,7 @@ func TestMain(t *testing.T) {
 	testMain(
 		t,
 		"../../example",
-		[]string{"--exec", "../scripts/simple.sh", "--exec-timeout", "1", "./..."},
+		[]string{"--exec", "../scripts/test-mutated-package.sh", "--exec-timeout", "1", "./..."},
 		returnOk,
 		"The mutation score is 0.538462 (7 passed, 6 failed, 1 skipped, total is 14)",
 	)
@@ -23,7 +23,7 @@ func TestMainMatch(t *testing.T) {
 	testMain(
 		t,
 		"../../example",
-		[]string{"--exec", "../scripts/simple.sh", "--exec-timeout", "1", "--match", "baz", "./..."},
+		[]string{"--exec", "../scripts/test-mutated-package.sh", "--exec-timeout", "1", "--match", "baz", "./..."},
 		returnOk,
 		"The mutation score is 0.000000 (0 passed, 1 failed, 0 skipped, total is 1)",
 	)

scripts/simple.sh → scripts/test-current-directory.sh

@@ -1,5 +1,14 @@
 #!/bin/bash
 
+# This exec script implements
+# - the replacement of the original file with the mutation,
+# - the execution of all tests originating from the current directory,
+# - and the reporting if the mutation was killed.
+
+if [ -z ${MUTATE_CHANGED+x} ]; then echo "MUTATE_CHANGED is not set"; exit 1; fi
+if [ -z ${MUTATE_ORIGINAL+x} ]; then echo "MUTATE_ORIGINAL is not set"; exit 1; fi
+if [ -z ${MUTATE_PACKAGE+x} ]; then echo "MUTATE_PACKAGE is not set"; exit 1; fi
+
 function clean_up {
 	if [ -f $MUTATE_ORIGINAL.tmp ];
 	then
@@ -21,7 +30,11 @@ cp $MUTATE_CHANGED $MUTATE_ORIGINAL
 
 export MUTATE_TIMEOUT=${MUTATE_TIMEOUT:-10}
 
-GOMUTESTING_TEST=$(go test -timeout $(printf '%ds' $MUTATE_TIMEOUT) ./... 2>&1)
+if [ -n "$TEST_RECURSIVE" ]; then
+	TEST_RECURSIVE="/..."
+fi
+
+GOMUTESTING_TEST=$(go test -timeout $(printf '%ds' $MUTATE_TIMEOUT) .$TEST_RECURSIVE 2>&1)
 export GOMUTESTING_RESULT=$?
 
 if [ "$MUTATE_DEBUG" = true ] ; then

scripts/test-mutated-package.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+
+# This exec script implements
+# - the replacement of the original file with the mutation,
+# - the execution of all tests originating from the package of the mutated file,
+# - and the reporting if the mutation was killed.
+
+if [ -z ${MUTATE_CHANGED+x} ]; then echo "MUTATE_CHANGED is not set"; exit 1; fi
+if [ -z ${MUTATE_ORIGINAL+x} ]; then echo "MUTATE_ORIGINAL is not set"; exit 1; fi
+if [ -z ${MUTATE_PACKAGE+x} ]; then echo "MUTATE_PACKAGE is not set"; exit 1; fi
+
+function clean_up {
+	if [ -f $MUTATE_ORIGINAL.tmp ];
+	then
+		mv $MUTATE_ORIGINAL.tmp $MUTATE_ORIGINAL
+	fi
+}
+
+function sig_handler {
+	clean_up
+
+	exit $GOMUTESTING_RESULT
+}
+trap sig_handler SIGHUP SIGINT SIGTERM
+
+export GOMUTESTING_DIFF=$(diff -u $MUTATE_ORIGINAL $MUTATE_CHANGED)
+
+mv $MUTATE_ORIGINAL $MUTATE_ORIGINAL.tmp
+cp $MUTATE_CHANGED $MUTATE_ORIGINAL
+
+export MUTATE_TIMEOUT=${MUTATE_TIMEOUT:-10}
+
+if [ -n "$TEST_RECURSIVE" ]; then
+	TEST_RECURSIVE="/..."
+fi
+
+GOMUTESTING_TEST=$(go test -timeout $(printf '%ds' $MUTATE_TIMEOUT) $MUTATE_PACKAGE$TEST_RECURSIVE 2>&1)
+export GOMUTESTING_RESULT=$?
+
+if [ "$MUTATE_DEBUG" = true ] ; then
+	echo "$GOMUTESTING_TEST"
+fi
+
+clean_up
+
+case $GOMUTESTING_RESULT in
+0) # tests passed -> FAIL
+	echo "$GOMUTESTING_DIFF"
+
+	exit 1
+	;;
+1) # tests failed -> PASS
+	if [ "$MUTATE_DEBUG" = true ] ; then
+		echo "$GOMUTESTING_DIFF"
+	fi
+
+	exit 0
+	;;
+2) # did not compile -> SKIP
+	if [ "$MUTATE_VERBOSE" = true ] ; then
+		echo "Mutation did not compile"
+	fi
+
+	if [ "$MUTATE_DEBUG" = true ] ; then
+		echo "$GOMUTESTING_DIFF"
+	fi
+
+	exit 2
+	;;
+*) # Unkown exit code -> SKIP
+	echo "Unknown exit code"
+	echo "$GOMUTESTING_DIFF"
+
+	exit $GOMUTESTING_RESULT
+	;;
+esac