Revert of revert with downgrade to Go1.19 (#102)

* Bring changes back except go 1.20 test changes

* Downgrade to Go 1.19

* Make output of test added after 1.20 to match output of 1.19

* Update setup-go

.github/workflows/test.yml

@@ -3,55 +3,53 @@ name: Run linter and tests
 on: [push, pull_request, workflow_dispatch]
 
 jobs:
-
   lint:
     runs-on: ubuntu-latest
     steps:
-    - name: Install Go
-      uses: actions/setup-go@b22fbbc2921299758641fab08929b4ac52b32923
-      with:
-        go-version: 1.18.x
-    - name: Checkout code
-      uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
-    - name: Run linters
-      uses: golangci/golangci-lint-action@v3
-      with:
-        version: v1.51
+      - name: Install Go
+        uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9
+        with:
+          go-version: 1.19.x
+      - name: Checkout code
+        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+      - name: Run linters
+        uses: golangci/golangci-lint-action@v3
+        with:
+          version: v1.51
 
   test:
     strategy:
       matrix:
-        go-version: [1.18.x]
         platform: [ubuntu-latest, windows-latest]
     runs-on: ${{ matrix.platform }}
     steps:
-    - name: Install Go
-      if: success()
-      uses: actions/setup-go@b22fbbc2921299758641fab08929b4ac52b32923
-      with:
-        go-version: 1.18.x
-    - name: Checkout code
-      uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
-    - name: Run tests
-      run: go test ./... -v -covermode=count
+      - name: Install Go
+        if: success()
+        uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9
+        with:
+          go-version: 1.19.x
+      - name: Checkout code
+        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+      - name: Run tests
+        run: go test ./... -v -covermode=count
 
   coverage:
     runs-on: ubuntu-latest
     steps:
-    - name: Install Go
-      if: success()
-      uses: actions/setup-go@b22fbbc2921299758641fab08929b4ac52b32923
-      with:
-        go-version: 1.18.x
-    - name: Checkout code
-      uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
-    - name: Calc coverage
-      run: |
-        go test ./... -v -covermode=count -coverprofile=coverage.out
-    - name: Convert coverage.out to coverage.lcov
-      uses: jandelgado/gcov2lcov-action@c680c0f7c7442485f1749eb2a13e54a686e76eb5
-    - name: Coveralls
-      uses: coverallsapp/github-action@9ba913c152ae4be1327bfb9085dc806cedb44057
-      with:
+      - name: Install Go
+        if: success()
+        uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9
+        with:
+          go-version: 1.19.x
+      - name: Checkout code
+        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+      - name: Calc coverage
+        run: |
+          go test ./... -v -covermode=count -coverprofile=coverage.out
+      - name: Convert coverage.out to coverage.lcov
+        uses: jandelgado/gcov2lcov-action@c680c0f7c7442485f1749eb2a13e54a686e76eb5
+      - name: Coveralls
+        uses: coverallsapp/github-action@9ba913c152ae4be1327bfb9085dc806cedb44057
+        with:
           github-token: ${{ secrets.github_token }}
           path-to-lcov: coverage.lcov

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.18-alpine3.15
+FROM golang:1.19.7-alpine3.17
 
 # add addtional packages needed for the race detector to work
 RUN apk add --update build-base make 

README.md

@@ -225,5 +225,13 @@ Otherwise no task ids will be set at all.
 
 Finding the task id is robust again other comments before or after or in the same line as the `testRunnerTaskID` comment, see the [conditionals-with-task-ids test file][task-id-comments-examples] for examples.
 
+## Known limitations
+
+Besides what is mentioned in the open issues, the test runner has the following limitations currently.
+
+- The test runner assumes all test functions `func Test...` can be found in one file.
+- The `cases_test.go` file is ignored when extracting the code for the test.
+- Sub-tests need to follow a certain format, see details above.
+
 [task-id]: https://exercism.org/docs/building/tooling/test-runners/interface#h-task-id
 [task-id-comments-examples]: https://github.com/exercism/go-test-runner/tree/main/testrunner/testdata/concept/conditionals-with-task-ids

external-packages/go.mod

@@ -1,8 +1,8 @@
 module external_packages
 
-go 1.16
+go 1.18
 
 require (
 	golang.org/x/exp v0.0.0-20221006183845-316c7553db56
-	golang.org/x/text v0.3.7
+	golang.org/x/text v0.7.0
 )

external-packages/go.sum

@@ -1,27 +1,4 @@
-github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/exp v0.0.0-20221006183845-316c7553db56 h1:BrYbdKcCNjLyrN6aKqXy4hPw9qGI8IATkj4EWv9Q+kQ=
 golang.org/x/exp v0.0.0-20221006183845-316c7553db56/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
-golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
-golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=

go.mod

@@ -1,6 +1,6 @@
 module github.com/exercism/go-test-runner
 
-go 1.18
+go 1.19
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect

integration_test.go

@@ -54,42 +54,46 @@ func TestIntegration(t *testing.T) {
 		{
 			// This test case covers the case the code under test does not compile,
 			// i.e. "go build ." would fail.
-			inputDir: "./testrunner/testdata/practice/broken",
-			expected: "./testrunner/testdata/expected/broken.json",
+			inputDir: filepath.Join("testrunner", "testdata", "practice", "broken"),
+			expected: filepath.Join("testrunner", "testdata", "expected", "broken.json"),
 		},
 		{
 			// This test case covers the case that the test code does not compile,
 			// i.e. "go build ." would succeed but "go test" returns compilation errors.
-			inputDir: "./testrunner/testdata/practice/missing_func",
-			expected: "./testrunner/testdata/expected/missing_func.json",
+			inputDir: filepath.Join("testrunner", "testdata", "practice", "missing_func"),
+			expected: filepath.Join("testrunner", "testdata", "expected", "missing_func.json"),
 		},
 		{
-			inputDir: "./testrunner/testdata/practice/broken_import",
-			expected: "./testrunner/testdata/expected/broken_import.json",
+			inputDir: filepath.Join("testrunner", "testdata", "practice", "broken_import"),
+			expected: filepath.Join("testrunner", "testdata", "expected", "broken_import.json"),
 		},
 		{
-			inputDir: "./testrunner/testdata/practice/passing",
-			expected: "./testrunner/testdata/expected/passing.json",
+			inputDir: filepath.Join("testrunner", "testdata", "practice", "passing"),
+			expected: filepath.Join("testrunner", "testdata", "expected", "passing.json"),
 		},
 		{
-			inputDir: "./testrunner/testdata/practice/pkg_level_error",
-			expected: "./testrunner/testdata/expected/pkg_level_error.json",
+			inputDir: filepath.Join("testrunner", "testdata", "practice", "pkg_level_error"),
+			expected: filepath.Join("testrunner", "testdata", "expected", "pkg_level_error.json"),
 		},
 		{
-			inputDir: "./testrunner/testdata/practice/failing",
-			expected: "./testrunner/testdata/expected/failing.json",
+			inputDir: filepath.Join("testrunner", "testdata", "practice", "failing"),
+			expected: filepath.Join("testrunner", "testdata", "expected", "failing.json"),
 		},
 		{
-			inputDir: "./testrunner/testdata/concept/auto_assigned_task_ids",
-			expected: "./testrunner/testdata/expected/auto_assigned_task_ids.json",
+			inputDir: filepath.Join("testrunner", "testdata", "concept", "auto_assigned_task_ids"),
+			expected: filepath.Join("testrunner", "testdata", "expected", "auto_assigned_task_ids.json"),
 		},
 		{
-			inputDir: "./testrunner/testdata/concept/explicit_task_ids",
-			expected: "./testrunner/testdata/expected/explicit_task_ids.json",
+			inputDir: filepath.Join("testrunner", "testdata", "concept", "explicit_task_ids"),
+			expected: filepath.Join("testrunner", "testdata", "expected", "explicit_task_ids.json"),
 		},
 		{
-			inputDir: "./testrunner/testdata/concept/missing_task_ids",
-			expected: "./testrunner/testdata/expected/missing_task_ids.json",
+			inputDir: filepath.Join("testrunner", "testdata", "concept", "missing_task_ids"),
+			expected: filepath.Join("testrunner", "testdata", "expected", "missing_task_ids.json"),
+		},
+		{
+			inputDir: filepath.Join("testrunner", "testdata", "concept", "non_executed_tests"),
+			expected: filepath.Join("testrunner", "testdata", "expected", "non_executed_tests.json"),
 		},
 	}
 
@@ -106,7 +110,7 @@ func TestIntegration(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.inputDir, func(t *testing.T) {
-			err := os.RemoveAll("./outdir")
+			err := os.RemoveAll("outdir")
 			require.NoError(t, err, "failed to clean up output directory")
 
 			var stdout, stderr bytes.Buffer
@@ -119,7 +123,7 @@ func TestIntegration(t *testing.T) {
 			err = cmd.Run()
 			require.NoErrorf(t, err, "failed to execute test runner: %s %s", stdout.String(), stderr.String())
 
-			resultBytes, err := os.ReadFile("./outdir/results.json")
+			resultBytes, err := os.ReadFile(filepath.Join("outdir", "results.json"))
 			require.NoError(t, err, "failed to read results")
 
 			result := sanitizeResult(string(resultBytes), []string{goExe, currentDir, goRoot})

main.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"fmt"
-	"io/ioutil"
 	"log"
 	"os"
 	"path/filepath"
@@ -23,7 +22,7 @@ func main() {
 
 	report := testrunner.Execute(input_dir)
 	results := filepath.Join(output_dir, "results.json")
-	err := ioutil.WriteFile(results, report, 0644)
+	err := os.WriteFile(results, report, 0644)
 	if err != nil {
 		log.Fatalf("Failed to write results.json: %s", err)
 	}

main_test.go

@@ -4,12 +4,17 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"testing"
 )
 
 func ExampleMain() {
-	os.Args = []string{"path", "testrunner/testdata/practice/passing", "outdir"}
+	os.Args = []string{
+		"path",
+		filepath.Join("testrunner", "testdata", "practice", "passing"),
+		"outdir",
+	}
 	main()
 	// Output:
 }
@@ -79,9 +84,9 @@ func TestCheckArgs(t *testing.T) {
 		{
 			name:       "broken output_dir",
 			input_dir:  "testrunner",
-			output_dir: "/tmp/broken/rmme",
+			output_dir: filepath.Join("/", "tmp", "broken", "rmme"),
 			ok:         false,
-			msg:        "output_dir /tmp/broken/rmme does not exist, mkdir failed:",
+			msg:        "output_dir " + filepath.Join("/", "tmp", "broken", "rmme") + " does not exist, mkdir failed:",
 		},
 	}
 	for _, tt := range tests {

testrunner/ast.go

@@ -29,25 +29,49 @@ type subTestAstInfo struct {
 	rangeAstIdx    int
 }
 
-// return the code of the "test" function from a file
-func getFuncCodeAndTaskID(test string, fstr string) (string, uint64) {
+type rootLevelTest struct {
+	name     string
+	fileName string
+	code     string
+	taskID   uint64
+}
+
+// FindAllRootLevelTests parses the test file and extracts the name,
+// test code and task id for each top level test (parent test) in the file.
+func FindAllRootLevelTests(fileName string) []rootLevelTest {
+	defer handleASTPanic()
+	tests := []rootLevelTest{}
 	fset := token.NewFileSet()
 	ppc := parser.ParseComments
-	file, err := parser.ParseFile(fset, fstr, nil, ppc)
+	file, err := parser.ParseFile(fset, fileName, nil, ppc)
 	if err != nil {
-		log.Printf("warning: '%s' not parsed from '%s': %s", test, fstr, err)
-		return "", 0
+		log.Printf("error: not able to parse '%s': %s", fileName, err)
+		return nil
 	}
 	for _, d := range file.Decls {
-		if f, ok := d.(*ast.FuncDecl); ok && f.Name.Name == test {
+		if f, ok := d.(*ast.FuncDecl); ok && strings.HasPrefix(f.Name.Name, "Test") {
 			taskID := findTaskID(f.Doc)
 			fun := &printer.CommentedNode{Node: f, Comments: file.Comments}
 			var buf bytes.Buffer
 			printer.Fprint(&buf, fset, fun)
-			return buf.String(), taskID
+
+			tests = append(tests, rootLevelTest{
+				name:     f.Name.Name,
+				fileName: fileName,
+				code:     buf.String(),
+				taskID:   taskID,
+			})
 		}
 	}
-	return "", 0
+	return tests
+}
+
+func ConvertToMapByTestName(tests []rootLevelTest) map[string]rootLevelTest {
+	result := map[string]rootLevelTest{}
+	for i := range tests {
+		result[tests[i].name] = tests[i]
+	}
+	return result
 }
 
 var taskIDFormat = regexp.MustCompile(`testRunnerTaskID=([0-9]+)`)

testrunner/ast_test.go

@@ -1,6 +1,7 @@
 package testrunner
 
 import (
+	"path/filepath"
 	"testing"
 )
 
@@ -14,25 +15,23 @@ func TestGetFuncCode(t *testing.T) {
 		{
 			name:     "valid call",
 			testName: "TestNonSubtest",
-			testFile: "testdata/concept/conditionals/conditionals_test.go",
+			testFile: filepath.Join("testdata", "concept", "conditionals", "conditionals_test.go"),
 			code:     "func TestNonSubtest(t *testing.T) {\n\t// comments should be included\n\tfmt.Println(\"the whole block\")\n\tfmt.Println(\"should be returned\")\n}",
-		}, {
-			name:     "missing test",
-			testName: "TestNothing",
-			testFile: "testdata/concept/conditionals/conditionals_test.go",
-			code:     "",
-		}, {
+		},
+		{
 			name:     "invalid test file",
 			testName: "TestNonSubtest",
-			testFile: "testdata/concept/conditionals/conditionals_missing.go",
+			testFile: filepath.Join("testdata", "concept", "conditionals", "conditionals_missing.go"),
 			code:     "",
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if code, _ := getFuncCodeAndTaskID(tt.testName, tt.testFile); code != tt.code {
-				t.Errorf("getFuncCode(%v, %v) = %v; want %v",
-					tt.testName, tt.testFile, code, tt.code)
+			rootLevelTests := FindAllRootLevelTests(tt.testFile)
+			rootLevelTestsMap := ConvertToMapByTestName(rootLevelTests)
+			if rootLevelTestsMap[tt.testName].code != tt.code {
+				t.Errorf("FindAllRootLevelTests for %s did not return correct code, got %v; want %v",
+					tt.testFile, rootLevelTestsMap[tt.testName].code, tt.code)
 			}
 		})
 	}