Do not ignore coverage count if there are failing tests

evaluate/repository.go

@@ -54,7 +54,8 @@ func Repository(logger *log.Logger, resultPath string, model evalmodel.Model, la
 		repositoryAssessment.Add(assessments)
 		repositoryAssessment.Award(metrics.AssessmentKeyResponseNoError)
 
-		coverage, err := language.Execute(log, testDataPath)
+		coverage, ps, err := language.Execute(log, testDataPath)
+		problems = append(problems, ps...)
 		if err != nil {
 			problems = append(problems, pkgerrors.WithMessage(err, filePath))
 

language/golang/language.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 	"regexp"
+	"strconv"
 	"strings"
 
 	pkgerrors "github.com/pkg/errors"
@@ -74,14 +75,10 @@ func (l *Language) TestFramework() (testFramework string) {
 	return ""
 }
 
-var languageGoNoTestsMatch = regexp.MustCompile(`(?m)^DONE (\d+) tests.*in (.+?)$`)
-var languageGoCoverageMatch = regexp.MustCompile(`(?m)^coverage: (\d+\.?\d+)% of statements`)
-var languageGoNoCoverageMatch = regexp.MustCompile(`(?m)^coverage: \[no statements\]$`)
+var languageGoTestsErrorMatch = regexp.MustCompile(`DONE (\d+) tests, (\d+) error`)
 
 // Execute invokes the language specific testing on the given repository.
-func (l *Language) Execute(logger *log.Logger, repositoryPath string) (coverage uint64, err error) {
-	coverageFilePath := filepath.Join(repositoryPath, "coverage.json")
-
+func (l *Language) Execute(logger *log.Logger, repositoryPath string) (coverage uint64, problems []error, err error) {
 	commandOutput, err := util.CommandWithResult(context.Background(), logger, &util.Command{
 		Command: []string{
 			"go",
@@ -92,9 +89,10 @@ func (l *Language) Execute(logger *log.Logger, repositoryPath string) (coverage
 		Directory: repositoryPath,
 	})
 	if err != nil {
-		return 0, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)
+		return 0, problems, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)
 	}
 
+	coverageFilePath := filepath.Join(repositoryPath, "coverage.json")
 	commandOutput, err = util.CommandWithResult(context.Background(), logger, &util.Command{
 		Command: []string{
 			tools.SymflowerPath, "test",
@@ -106,8 +104,33 @@ func (l *Language) Execute(logger *log.Logger, repositoryPath string) (coverage
 		Directory: repositoryPath,
 	})
 	if err != nil {
-		return 0, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)
+		if strings.Contains(commandOutput, "ERROR: no test files found") {
+			return 0, problems, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)
+		}
+
+		testSummary := languageGoTestsErrorMatch.FindStringSubmatch(commandOutput)
+		if len(testSummary) > 0 {
+			testCount, e := strconv.Atoi(testSummary[1])
+			if e != nil {
+				return 0, problems, pkgerrors.WithStack(e)
+			}
+			errorCount, e := strconv.Atoi(testSummary[2])
+			if e != nil {
+				return 0, problems, pkgerrors.WithStack(e)
+			}
+			// If there are no executed tests but only errors, then we assume there is a syntax error.
+			if testCount == 0 && errorCount > 0 {
+				return 0, problems, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)
+			}
+		}
+
+		problems = append(problems, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput))
+	}
+
+	coverage, err = language.CoverageObjectCountOfFile(coverageFilePath)
+	if err != nil {
+		return 0, problems, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)
 	}
 
-	return language.CoverageObjectCountOfFile(coverageFilePath)
+	return coverage, problems, nil
 }

language/golang/language_test.go

@@ -64,9 +64,10 @@ func TestLanguageExecute(t *testing.T) {
 		RepositoryPath   string
 		RepositoryChange func(t *testing.T, repositoryPath string)
 
-		ExpectedCoverage  uint64
-		ExpectedError     error
-		ExpectedErrorText string
+		ExpectedCoverage     uint64
+		ExpectedProblemTexts []string
+		ExpectedError        error
+		ExpectedErrorText    string
 	}
 
 	validate := func(t *testing.T, tc *testCase) {
@@ -89,7 +90,12 @@ func TestLanguageExecute(t *testing.T) {
 			if tc.Language == nil {
 				tc.Language = &Language{}
 			}
-			actualCoverage, actualError := tc.Language.Execute(logger, repositoryPath)
+			actualCoverage, actualProblems, actualError := tc.Language.Execute(logger, repositoryPath)
+
+			require.Equal(t, len(tc.ExpectedProblemTexts), len(actualProblems), "the number of expected problems need to match the number of actual problems")
+			for i, expectedProblemText := range tc.ExpectedProblemTexts {
+				assert.ErrorContains(t, actualProblems[i], expectedProblemText)
+			}
 
 			if tc.ExpectedError != nil {
 				assert.ErrorIs(t, actualError, tc.ExpectedError)
@@ -107,7 +113,8 @@ func TestLanguageExecute(t *testing.T) {
 
 		RepositoryPath: filepath.Join("..", "..", "testdata", "golang", "plain"),
 
-		ExpectedCoverage: 0,
+		ExpectedCoverage:  0,
+		ExpectedErrorText: "exit status 1",
 	})
 
 	t.Run("With test file", func(t *testing.T) {
@@ -132,6 +139,45 @@ func TestLanguageExecute(t *testing.T) {
 			ExpectedCoverage: 1,
 		})
 
+		validate(t, &testCase{
+			Name: "Failing tests",
+
+			RepositoryPath: filepath.Join("..", "..", "testdata", "golang", "light"),
+			RepositoryChange: func(t *testing.T, repositoryPath string) {
+				require.NoError(t, os.WriteFile(filepath.Join(repositoryPath, "simpleIfElse_test.go"), []byte(bytesutil.StringTrimIndentations(`
+					package light
+
+					import (
+						"testing"
+					)
+
+					func TestSimpleIfElse(t *testing.T) {
+						type args struct {
+							i int
+						}
+						tests := []struct {
+								name string
+								args
+								want int
+						}{
+								{"i equal 1", args{1}, 0},
+								{"i equal 2", args{2}, 0}, // This is an invalid test case.
+						}
+						for _, tt := range tests {
+								if got := simpleIfElse(tt.args.i); got != tt.want {
+										t.Errorf("%q. simpleIfElse(%v) = %v, want %v", tt.name, tt.args.i, got, tt.want)
+								}
+						}
+					}
+				`)), 0660))
+			},
+
+			ExpectedCoverage: 2,
+			ExpectedProblemTexts: []string{
+				"exit status 1", // Test execution fails.
+			},
+		})
+
 		validate(t, &testCase{
 			Name: "Syntax error",
 

language/java/language.go

@@ -89,7 +89,7 @@ func (l *Language) TestFramework() (testFramework string) {
 var languageJavaCoverageMatch = regexp.MustCompile(`Total coverage (.+?)%`)
 
 // Execute invokes the language specific testing on the given repository.
-func (l *Language) Execute(logger *log.Logger, repositoryPath string) (coverage uint64, err error) {
+func (l *Language) Execute(logger *log.Logger, repositoryPath string) (coverage uint64, problems []error, err error) {
 	coverageFilePath := filepath.Join(repositoryPath, "coverage.json")
 	commandOutput, err := util.CommandWithResult(context.Background(), logger, &util.Command{
 		Command: []string{
@@ -102,8 +102,13 @@ func (l *Language) Execute(logger *log.Logger, repositoryPath string) (coverage
 		Directory: repositoryPath,
 	})
 	if err != nil {
-		return 0, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)
+		return 0, nil, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)
 	}
 
-	return language.CoverageObjectCountOfFile(coverageFilePath)
+	coverage, err = language.CoverageObjectCountOfFile(coverageFilePath)
+	if err != nil {
+		return 0, nil, pkgerrors.WithMessage(pkgerrors.WithStack(err), commandOutput)
+	}
+
+	return coverage, nil, nil
 }

language/java/language_test.go

@@ -130,9 +130,10 @@ func TestLanguageExecute(t *testing.T) {
 		RepositoryPath   string
 		RepositoryChange func(t *testing.T, repositoryPath string)
 
-		ExpectedCoverage  uint64
-		ExpectedError     error
-		ExpectedErrorText string
+		ExpectedCoverage     uint64
+		ExpectedProblemTexts []string
+		ExpectedError        error
+		ExpectedErrorText    string
 	}
 
 	validate := func(t *testing.T, tc *testCase) {
@@ -155,7 +156,12 @@ func TestLanguageExecute(t *testing.T) {
 			if tc.Language == nil {
 				tc.Language = &Language{}
 			}
-			actualCoverage, actualError := tc.Language.Execute(logger, repositoryPath)
+			actualCoverage, actualProblems, actualError := tc.Language.Execute(logger, repositoryPath)
+
+			require.Equal(t, len(tc.ExpectedProblemTexts), len(actualProblems), "the number of expected problems need to match the number of actual problems")
+			for i, expectedProblemText := range tc.ExpectedProblemTexts {
+				assert.ErrorContains(t, actualProblems[i], expectedProblemText)
+			}
 
 			if tc.ExpectedError != nil {
 				assert.ErrorIs(t, actualError, tc.ExpectedError)
@@ -173,7 +179,8 @@ func TestLanguageExecute(t *testing.T) {
 
 		RepositoryPath: filepath.Join("..", "..", "testdata", "java", "plain"),
 
-		ExpectedCoverage: 0, // TODO Let the test case identify and error that there are no test files (needs to be implemented in `symflower test`). https://github.com/symflower/eval-dev-quality/issues/35
+		ExpectedCoverage:  0,
+		ExpectedErrorText: "exit status 1",
 	})
 
 	t.Run("With test file", func(t *testing.T) {
@@ -201,6 +208,42 @@ func TestLanguageExecute(t *testing.T) {
 			ExpectedCoverage: 1,
 		})
 
+		validate(t, &testCase{
+			Name: "Failing tests",
+
+			RepositoryPath: filepath.Join("..", "..", "testdata", "java", "light"),
+			RepositoryChange: func(t *testing.T, repositoryPath string) {
+				javaTestFilePath := filepath.Join(repositoryPath, "src/test/java/com/eval/SimpleIfElseSymflowerTest.java")
+				require.NoError(t, os.MkdirAll(filepath.Dir(javaTestFilePath), 0755))
+				require.NoError(t, os.WriteFile(javaTestFilePath, []byte(bytesutil.StringTrimIndentations(`
+					package com.eval;
+
+					import org.junit.jupiter.api.*;
+
+					public class SimpleIfElseSymflowerTest {
+						@Test
+						public void simpleIfElse1() {
+							int expected = 0;
+							int actual = SimpleIfElse.simpleIfElse(1);
+
+							Assertions.assertEquals(expected, actual);
+						}
+
+						@Test
+						public void simpleIfElse2() {
+							// This is an invalid test case.
+							int expected = 0;
+							int actual = SimpleIfElse.simpleIfElse(2);
+
+							Assertions.assertEquals(expected, actual);
+						}
+					}
+				`)), 0660))
+			},
+
+			ExpectedCoverage: 5,
+		})
+
 		validate(t, &testCase{
 			Name: "Syntax error",
 

language/language.go

@@ -23,7 +23,7 @@ type Language interface {
 	TestFramework() (testFramework string)
 
 	// Execute invokes the language specific testing on the given repository.
-	Execute(logger *log.Logger, repositoryPath string) (coverage uint64, err error)
+	Execute(logger *log.Logger, repositoryPath string) (coverage uint64, problems []error, err error)
 }
 
 // Languages holds a register of all languages.

model/symflower/symflower_test.go

@@ -62,8 +62,9 @@ func TestModelGenerateTestsForFile(t *testing.T) {
 			}
 			metricstesting.AssertAssessmentsEqual(t, tc.ExpectedAssessment, actualAssessment)
 
-			actualCoverage, err := tc.Language.Execute(logger, repositoryPath)
+			actualCoverage, actualProblems, err := tc.Language.Execute(logger, repositoryPath)
 			require.NoError(t, err)
+			require.Empty(t, actualProblems)
 			assert.Equal(t, tc.ExpectedCoverage, actualCoverage)
 		})
 	}