Do not ignore coverage count if there are failing tests

evaluate/repository.go

@@ -54,7 +54,8 @@ func Repository(logger *log.Logger, resultPath string, model evalmodel.Model, la
 		repositoryAssessment.Add(assessments)
 		repositoryAssessment.Award(metrics.AssessmentKeyResponseNoError)
 
-		coverage, err := language.Execute(log, testDataPath)
+		coverage, ps, err := language.Execute(log, testDataPath)
+		problems = append(problems, ps...)
 		if err != nil {
 			problems = append(problems, pkgerrors.WithMessage(err, filePath))
 

language/coverage/coverage.go

@@ -55,7 +55,7 @@ func CoverageObjectCountOfFile(coverageFilePath string) (coverageObjectCount uin
 }
 
 // ExecuteSymflowerTest invokes the language specific testing on the given repository using "symflower test".
-func ExecuteSymflowerTest(language language.Language, logger *log.Logger, repositoryPath string) (coverage uint64, err error) {
+func ExecuteSymflowerTest(language language.Language, logger *log.Logger, repositoryPath string) (coverage uint64, problems []error, err error) {
 	coverageFilePath := filepath.Join(repositoryPath, "coverage.json")
 	commandOutput, err := util.CommandWithResult(context.Background(), logger, &util.Command{
 		Command: []string{
@@ -68,8 +68,13 @@ func ExecuteSymflowerTest(language language.Language, logger *log.Logger, reposi
 		Directory: repositoryPath,
 	})
 	if err != nil {
-		return 0, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)
+		problems = append(problems, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput))
 	}
 
-	return CoverageObjectCountOfFile(coverageFilePath)
+	coverage, err = CoverageObjectCountOfFile(coverageFilePath)
+	if err != nil {
+		problems = append(problems, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput))
+	}
+
+	return coverage, problems, nil
 }

language/golang/language.go

@@ -79,7 +79,7 @@ var languageGoCoverageMatch = regexp.MustCompile(`(?m)^coverage: (\d+\.?\d+)% of
 var languageGoNoCoverageMatch = regexp.MustCompile(`(?m)^coverage: \[no statements\]$`)
 
 // Execute invokes the language specific testing on the given repository.
-func (l *Language) Execute(logger *log.Logger, repositoryPath string) (coverageCount uint64, err error) {
+func (l *Language) Execute(logger *log.Logger, repositoryPath string) (coverageCount uint64, problems []error, err error) {
 	commandOutput, err := util.CommandWithResult(context.Background(), logger, &util.Command{
 		Command: []string{
 			"go",
@@ -90,7 +90,7 @@ func (l *Language) Execute(logger *log.Logger, repositoryPath string) (coverageC
 		Directory: repositoryPath,
 	})
 	if err != nil {
-		return 0, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)
+		return 0, problems, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)
 	}
 
 	return coverage.ExecuteSymflowerTest(l, logger, repositoryPath)

language/golang/language_test.go

@@ -64,9 +64,10 @@ func TestLanguageExecute(t *testing.T) {
 		RepositoryPath   string
 		RepositoryChange func(t *testing.T, repositoryPath string)
 
-		ExpectedCoverage  uint64
-		ExpectedError     error
-		ExpectedErrorText string
+		ExpectedCoverage     uint64
+		ExpectedProblemTexts []string
+		ExpectedError        error
+		ExpectedErrorText    string
 	}
 
 	validate := func(t *testing.T, tc *testCase) {
@@ -89,7 +90,12 @@ func TestLanguageExecute(t *testing.T) {
 			if tc.Language == nil {
 				tc.Language = &Language{}
 			}
-			actualCoverage, actualError := tc.Language.Execute(logger, repositoryPath)
+			actualCoverage, actualProblems, actualError := tc.Language.Execute(logger, repositoryPath)
+
+			require.Equal(t, len(tc.ExpectedProblemTexts), len(actualProblems), "the number of expected problems need to match the number of actual problems")
+			for i, expectedProblemText := range tc.ExpectedProblemTexts {
+				assert.ErrorContains(t, actualProblems[i], expectedProblemText)
+			}
 
 			if tc.ExpectedError != nil {
 				assert.ErrorIs(t, actualError, tc.ExpectedError)
@@ -108,6 +114,10 @@ func TestLanguageExecute(t *testing.T) {
 		RepositoryPath: filepath.Join("..", "..", "testdata", "golang", "plain"),
 
 		ExpectedCoverage: 0,
+		ExpectedProblemTexts: []string{
+			"exit status 1",             // Test execution fails.
+			"no such file or directory", // No coverage file exists.
+		},
 	})
 
 	t.Run("With test file", func(t *testing.T) {
@@ -117,21 +127,60 @@ func TestLanguageExecute(t *testing.T) {
 			RepositoryPath: filepath.Join("..", "..", "testdata", "golang", "plain"),
 			RepositoryChange: func(t *testing.T, repositoryPath string) {
 				require.NoError(t, os.WriteFile(filepath.Join(repositoryPath, "plain_test.go"), []byte(bytesutil.StringTrimIndentations(`
-					package plain
+						package plain
 
-					import (
-						"testing"
-					)
+						import (
+							"testing"
+						)
 
-					func TestPlain(t *testing.T) {
-						plain()
-					}
-				`)), 0660))
+						func TestPlain(t *testing.T) {
+							plain()
+						}
+					`)), 0660))
 			},
 
 			ExpectedCoverage: 1,
 		})
 
+		validate(t, &testCase{
+			Name: "Failing tests",
+
+			RepositoryPath: filepath.Join("..", "..", "testdata", "golang", "light"),
+			RepositoryChange: func(t *testing.T, repositoryPath string) {
+				require.NoError(t, os.WriteFile(filepath.Join(repositoryPath, "simpleIfElse_test.go"), []byte(bytesutil.StringTrimIndentations(`
+						package light
+
+						import (
+							"testing"
+						)
+
+						func TestSimpleIfElse(t *testing.T) {
+							type args struct {
+								i int
+							}
+							tests := []struct {
+									name string
+									args
+									want int
+							}{
+									{"i equal 1", args{1}, 0},
+									{"i equal 2", args{2}, 0},
+							}
+							for _, tt := range tests {
+									if got := simpleIfElse(tt.args.i); got != tt.want {
+											t.Errorf("%q. simpleIfElse(%v) = %v, want %v", tt.name, tt.args.i, got, tt.want)
+									}
+							}
+						}
+					`)), 0660))
+			},
+
+			ExpectedCoverage: 2,
+			ExpectedProblemTexts: []string{
+				"exit status 1", // Test execution fails.
+			},
+		})
+
 		validate(t, &testCase{
 			Name: "Syntax error",
 
@@ -142,7 +191,10 @@ func TestLanguageExecute(t *testing.T) {
 				`)), 0660))
 			},
 
-			ExpectedErrorText: "exit status 1",
+			ExpectedProblemTexts: []string{
+				"exit status 1",             // Test execution fails.
+				"no such file or directory", // No coverage file exists.
+			},
 		})
 	})
 }

language/java/language.go

@@ -87,6 +87,6 @@ func (l *Language) TestFramework() (testFramework string) {
 var languageJavaCoverageMatch = regexp.MustCompile(`Total coverage (.+?)%`)
 
 // Execute invokes the language specific testing on the given repository.
-func (l *Language) Execute(logger *log.Logger, repositoryPath string) (coverageCount uint64, err error) {
+func (l *Language) Execute(logger *log.Logger, repositoryPath string) (coverageCount uint64, problems []error, err error) {
 	return coverage.ExecuteSymflowerTest(l, logger, repositoryPath)
 }

language/java/language_test.go

@@ -130,9 +130,10 @@ func TestLanguageExecute(t *testing.T) {
 		RepositoryPath   string
 		RepositoryChange func(t *testing.T, repositoryPath string)
 
-		ExpectedCoverage  uint64
-		ExpectedError     error
-		ExpectedErrorText string
+		ExpectedCoverage     uint64
+		ExpectedProblemTexts []string
+		ExpectedError        error
+		ExpectedErrorText    string
 	}
 
 	validate := func(t *testing.T, tc *testCase) {
@@ -155,7 +156,12 @@ func TestLanguageExecute(t *testing.T) {
 			if tc.Language == nil {
 				tc.Language = &Language{}
 			}
-			actualCoverage, actualError := tc.Language.Execute(logger, repositoryPath)
+			actualCoverage, actualProblems, actualError := tc.Language.Execute(logger, repositoryPath)
+
+			require.Equal(t, len(tc.ExpectedProblemTexts), len(actualProblems), "the number of expected problems need to match the number of actual problems")
+			for i, expectedProblemText := range tc.ExpectedProblemTexts {
+				assert.ErrorContains(t, actualProblems[i], expectedProblemText)
+			}
 
 			if tc.ExpectedError != nil {
 				assert.ErrorIs(t, actualError, tc.ExpectedError)
@@ -173,7 +179,11 @@ func TestLanguageExecute(t *testing.T) {
 
 		RepositoryPath: filepath.Join("..", "..", "testdata", "java", "plain"),
 
-		ExpectedCoverage: 0, // TODO Let the test case identify and error that there are no test files (needs to be implemented in `symflower test`). https://github.com/symflower/eval-dev-quality/issues/35
+		ExpectedCoverage: 0,
+		ExpectedProblemTexts: []string{
+			"exit status 1",             // Test execution fails.
+			"no such file or directory", // No coverage file exists.
+		},
 	})
 
 	t.Run("With test file", func(t *testing.T) {
@@ -201,6 +211,41 @@ func TestLanguageExecute(t *testing.T) {
 			ExpectedCoverage: 1,
 		})
 
+		validate(t, &testCase{
+			Name: "Failing tests",
+
+			RepositoryPath: filepath.Join("..", "..", "testdata", "java", "light"),
+			RepositoryChange: func(t *testing.T, repositoryPath string) {
+				javaTestFilePath := filepath.Join(repositoryPath, "src/test/java/com/eval/SimpleIfElseSymflowerTest.java")
+				require.NoError(t, os.MkdirAll(filepath.Dir(javaTestFilePath), 0755))
+				require.NoError(t, os.WriteFile(javaTestFilePath, []byte(bytesutil.StringTrimIndentations(`
+					package com.eval;
+
+					import org.junit.jupiter.api.*;
+
+					public class SimpleIfElseSymflowerTest {
+						@Test
+						public void simpleIfElse1() {
+							int expected = 0;
+							int actual = SimpleIfElse.simpleIfElse(1);
+
+							Assertions.assertEquals(expected, actual);
+						}
+
+						@Test
+						public void simpleIfElse2() {
+							int expected = 0;
+							int actual = SimpleIfElse.simpleIfElse(2);
+
+							Assertions.assertEquals(expected, actual);
+						}
+					}
+				`)), 0660))
+			},
+
+			ExpectedCoverage: 5,
+		})
+
 		validate(t, &testCase{
 			Name: "Syntax error",
 
@@ -213,7 +258,10 @@ func TestLanguageExecute(t *testing.T) {
 				`)), 0660))
 			},
 
-			ExpectedErrorText: "exit status 1",
+			ExpectedProblemTexts: []string{
+				"exit status 1",             // Test execution fails.
+				"no such file or directory", // No coverage file exists.
+			},
 		})
 	})
 }

language/language.go

@@ -23,7 +23,7 @@ type Language interface {
 	TestFramework() (testFramework string)
 
 	// Execute invokes the language specific testing on the given repository.
-	Execute(logger *log.Logger, repositoryPath string) (coverage uint64, err error)
+	Execute(logger *log.Logger, repositoryPath string) (coverage uint64, problems []error, err error)
 }
 
 // Languages holds a register of all languages.

model/symflower/symflower_test.go

@@ -62,8 +62,9 @@ func TestModelGenerateTestsForFile(t *testing.T) {
 			}
 			metricstesting.AssertAssessmentsEqual(t, tc.ExpectedAssessment, actualAssessment)
 
-			actualCoverage, err := tc.Language.Execute(logger, repositoryPath)
+			actualCoverage, actualProblems, err := tc.Language.Execute(logger, repositoryPath)
 			require.NoError(t, err)
+			require.Empty(t, actualProblems)
 			assert.Equal(t, tc.ExpectedCoverage, actualCoverage)
 		})
 	}