Add smoke tests

.dockerignore

@@ -1,8 +1,11 @@
 bin/run-in-docker.sh
+bin/run-tests-in-docker.sh
+bin/run-tests.sh
 .appends
 .git
 .github
 .gitignore
 .gitattributes
 .dockerignore
 Dockerfile
+tests/

.github/workflows/test.yml

@@ -33,6 +33,30 @@ jobs:
       - name: Run tests
         run: go test ./... -v -covermode=count
 
+  smoke-tests:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@4c0219f9ac95b02789c1075625400b2acbff50b1
+        with:
+          install: true
+
+      - name: Build Docker image and store in cache
+        uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825
+        with:
+          context: .
+          push: false
+          load: true
+          tags: exercism/go-test-runner
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Run Tests in Docker
+        run: bin/run-tests-in-docker.sh
+
   coverage:
     runs-on: ubuntu-latest
     steps:

.gitignore

@@ -10,4 +10,4 @@ log/*
 
 outdir
 
-!bin/run-in-docker.sh
+!/bin/*.sh

bin/run-in-docker.sh

@@ -30,14 +30,15 @@ output_dir="${3%/}"
 mkdir -p "${output_dir}"
 
 # Build the Docker image
-docker build --rm -t exercism/test-runner .
+docker build --rm -t exercism/go-test-runner .
+
+# TODO: allow image to work with --read-only filesystem
 
 # Run the Docker image using the settings mimicking the production environment
 docker run \
     --rm \
     --network none \
-    --read-only \
     --mount type=bind,src="${input_dir}",dst=/solution \
     --mount type=bind,src="${output_dir}",dst=/output \
     --mount type=volume,target=/tmp \
-    exercism/test-runner "${slug}" /solution /output
+    exercism/go-test-runner "${slug}" /solution /output

bin/run-tests-in-docker.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env sh
+
+# Synopsis:
+# Test the test runner Docker image by running it against a predefined set of 
+# solutions with an expected output.
+# The test runner Docker image is built automatically.
+
+# Output:
+# Outputs the diff of the expected test results against the actual test results
+# generated by the test runner Docker image.
+
+# Example:
+# ./bin/run-tests-in-docker.sh
+
+# Stop executing when a command returns a non-zero return code
+set -e
+
+# Build the Docker image
+docker build --rm -t exercism/go-test-runner .
+
+# TODO: allow image to work with --read-only filesystem
+
+# Run the Docker image using the settings mimicking the production environment
+docker run \
+    --rm \
+    --network none \
+    --mount type=bind,src="${PWD}/tests",dst=/opt/test-runner/tests \
+    --mount type=volume,dst=/tmp \
+    --volume "${PWD}/bin/run-tests.sh:/opt/test-runner/bin/run-tests.sh" \
+    --workdir /opt/test-runner \
+    --entrypoint /opt/test-runner/bin/run-tests.sh \
+    exercism/go-test-runner

bin/run-tests.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env sh
+
+# Synopsis:
+# Test the test runner by running it against a predefined set of solutions 
+# with an expected output.
+
+# Output:
+# Outputs the diff of the expected test results against the actual test results
+# generated by the test runner.
+
+# Example:
+# ./bin/run-tests.sh
+
+exit_code=0
+
+# Iterate over all test directories
+for test_dir in tests/*; do
+    test_dir_name=$(basename "${test_dir}")
+    test_dir_path=$(realpath "${test_dir}")
+    results_file_path="${test_dir_path}/results.json"
+    expected_results_file_path="${test_dir_path}/expected_results.json"
+
+    bin/run.sh "${test_dir_name}" "${test_dir_path}" "${test_dir_path}"
+
+    # OPTIONAL: Normalize the results file
+    # If the results.json file contains information that changes between 
+    # different test runs (e.g. timing information or paths), you should normalize
+    # the results file to allow the diff comparison below to work as expected
+    # sed -i -E \
+    #   -e 's/Elapsed time: [0-9]+\.[0-9]+ seconds//g' \
+    #   -e "s~${test_dir_path}~/solution~g" \
+    #   "${results_file_path}"
+
+    echo "${test_dir_name}: comparing results.json to expected_results.json"
+    diff "${results_file_path}" "${expected_results_file_path}"
+
+    if [ $? -ne 0 ]; then
+        exit_code=1
+    fi
+done
+
+exit ${exit_code}

tests/all-fail/.meta/config.json

@@ -0,0 +1,44 @@
+{
+  "authors": [
+    "kytrinyx"
+  ],
+  "contributors": [
+    "alebaffa",
+    "bitfield",
+    "da-edra",
+    "ekingery",
+    "ferhatelmas",
+    "hilary",
+    "ilmanzo",
+    "k4rtik",
+    "leenipper",
+    "petertseng",
+    "robphoenix",
+    "sebito91",
+    "soniakeys",
+    "tleen",
+    "tompao",
+    "zabawaba99",
+    "eklatzer"
+  ],
+  "files": {
+    "solution": [
+      "leap.go"
+    ],
+    "test": [
+      "leap_test.go"
+    ],
+    "example": [
+      ".meta/example.go"
+    ],
+    "editor": [
+      "cases_test.go"
+    ],
+    "invalidator": [
+      "go.mod"
+    ]
+  },
+  "blurb": "Given a year, report if it is a leap year.",
+  "source": "CodeRanch Cattle Drive, Assignment 3",
+  "source_url": "https://coderanch.com/t/718816/Leap"
+}

tests/all-fail/.meta/tests.toml

@@ -0,0 +1,30 @@
+# This is an auto-generated file. Regular comments will be removed when this
+# file is regenerated. Regenerating will not touch any manually added keys,
+# so comments can be added in a "comment" key.
+
+[6466b30d-519c-438e-935d-388224ab5223]
+description = "year not divisible by 4 in common year"
+
+[ac227e82-ee82-4a09-9eb6-4f84331ffdb0]
+description = "year divisible by 2, not divisible by 4 in common year"
+
+[4fe9b84c-8e65-489e-970b-856d60b8b78e]
+description = "year divisible by 4, not divisible by 100 in leap year"
+
+[7fc6aed7-e63c-48f5-ae05-5fe182f60a5d]
+description = "year divisible by 4 and 5 is still a leap year"
+
+[78a7848f-9667-4192-ae53-87b30c9a02dd]
+description = "year divisible by 100, not divisible by 400 in common year"
+
+[9d70f938-537c-40a6-ba19-f50739ce8bac]
+description = "year divisible by 100 but not by 3 is still not a leap year"
+
+[42ee56ad-d3e6-48f1-8e3f-c84078d916fc]
+description = "year divisible by 400 in leap year"
+
+[57902c77-6fe9-40de-8302-587b5c27121e]
+description = "year divisible by 400 but not by 125 is still a leap year"
+
+[c30331f6-f9f6-4881-ad38-8ca8c12520c1]
+description = "year divisible by 200, not divisible by 400 in common year"

tests/all-fail/cases_test.go

@@ -0,0 +1,58 @@
+package leap
+
+// This is an auto-generated file. Do not change it manually. Run the generator to update the file.
+// See https://github.com/exercism/go#synchronizing-tests-and-instructions
+// Source: exercism/problem-specifications
+// Commit: a2c75d2 leap: fix typo (#1726)
+
+var testCases = []struct {
+	description string
+	year        int
+	expected    bool
+}{
+	{
+		description: "year not divisible by 4 in common year",
+		year:        2015,
+		expected:    false,
+	},
+	{
+		description: "year divisible by 2, not divisible by 4 in common year",
+		year:        1970,
+		expected:    false,
+	},
+	{
+		description: "year divisible by 4, not divisible by 100 in leap year",
+		year:        1996,
+		expected:    true,
+	},
+	{
+		description: "year divisible by 4 and 5 is still a leap year",
+		year:        1960,
+		expected:    true,
+	},
+	{
+		description: "year divisible by 100, not divisible by 400 in common year",
+		year:        2100,
+		expected:    false,
+	},
+	{
+		description: "year divisible by 100 but not by 3 is still not a leap year",
+		year:        1900,
+		expected:    false,
+	},
+	{
+		description: "year divisible by 400 is leap year",
+		year:        2000,
+		expected:    true,
+	},
+	{
+		description: "year divisible by 400 but not by 125 is still a leap year",
+		year:        2400,
+		expected:    true,
+	},
+	{
+		description: "year divisible by 200, not divisible by 400 in common year",
+		year:        1800,
+		expected:    false,
+	},
+}

tests/all-fail/expected_results.json

@@ -0,0 +1,60 @@
+{
+	"status": "fail",
+	"version": 3,
+	"tests": [
+		{
+			"name": "TestLeapYears/ year not divisible by 4 in common year",
+			"status": "fail",
+			"test_code": "func TestLeapYears(t *testing.T) {\n\tfor _, tc := range testCases {\n\t\tt.Run(tc.description, func(t *testing.T) {\n\t\t\tactual := IsLeapYear(tc.year)\n\t\t\tif actual != tc.expected {\n\t\t\t\tt.Fatalf(\"IsLeapYear(%d) = %t, want %t\", tc.year, actual, tc.expected)\n\t\t\t}\n\t\t})\n\t}\n}",
+			"message": "\n=== RUN   TestLeapYears/year_not_divisible_by_4_in_common_year\n\n    leap_test.go:10: IsLeapYear(2015) = true, want false\n\n--- FAIL: TestLeapYears/year_not_divisible_by_4_in_common_year (0.00s)\n"
+		},
+		{
+			"name": "TestLeapYears/ year divisible by 2, not divisible by 4 in common year",
+			"status": "fail",
+			"test_code": "func TestLeapYears(t *testing.T) {\n\tfor _, tc := range testCases {\n\t\tt.Run(tc.description, func(t *testing.T) {\n\t\t\tactual := IsLeapYear(tc.year)\n\t\t\tif actual != tc.expected {\n\t\t\t\tt.Fatalf(\"IsLeapYear(%d) = %t, want %t\", tc.year, actual, tc.expected)\n\t\t\t}\n\t\t})\n\t}\n}",
+			"message": "\n=== RUN   TestLeapYears/year_divisible_by_2,_not_divisible_by_4_in_common_year\n\n    leap_test.go:10: IsLeapYear(1970) = true, want false\n\n--- FAIL: TestLeapYears/year_divisible_by_2,_not_divisible_by_4_in_common_year (0.00s)\n"
+		},
+		{
+			"name": "TestLeapYears/ year divisible by 4, not divisible by 100 in leap year",
+			"status": "fail",
+			"test_code": "func TestLeapYears(t *testing.T) {\n\tfor _, tc := range testCases {\n\t\tt.Run(tc.description, func(t *testing.T) {\n\t\t\tactual := IsLeapYear(tc.year)\n\t\t\tif actual != tc.expected {\n\t\t\t\tt.Fatalf(\"IsLeapYear(%d) = %t, want %t\", tc.year, actual, tc.expected)\n\t\t\t}\n\t\t})\n\t}\n}",
+			"message": "\n=== RUN   TestLeapYears/year_divisible_by_4,_not_divisible_by_100_in_leap_year\n\n    leap_test.go:10: IsLeapYear(1996) = false, want true\n\n--- FAIL: TestLeapYears/year_divisible_by_4,_not_divisible_by_100_in_leap_year (0.00s)\n"
+		},
+		{
+			"name": "TestLeapYears/ year divisible by 4 and 5 is still a leap year",
+			"status": "fail",
+			"test_code": "func TestLeapYears(t *testing.T) {\n\tfor _, tc := range testCases {\n\t\tt.Run(tc.description, func(t *testing.T) {\n\t\t\tactual := IsLeapYear(tc.year)\n\t\t\tif actual != tc.expected {\n\t\t\t\tt.Fatalf(\"IsLeapYear(%d) = %t, want %t\", tc.year, actual, tc.expected)\n\t\t\t}\n\t\t})\n\t}\n}",
+			"message": "\n=== RUN   TestLeapYears/year_divisible_by_4_and_5_is_still_a_leap_year\n\n    leap_test.go:10: IsLeapYear(1960) = false, want true\n\n--- FAIL: TestLeapYears/year_divisible_by_4_and_5_is_still_a_leap_year (0.00s)\n"
+		},
+		{
+			"name": "TestLeapYears/ year divisible by 100, not divisible by 400 in common year",
+			"status": "fail",
+			"test_code": "func TestLeapYears(t *testing.T) {\n\tfor _, tc := range testCases {\n\t\tt.Run(tc.description, func(t *testing.T) {\n\t\t\tactual := IsLeapYear(tc.year)\n\t\t\tif actual != tc.expected {\n\t\t\t\tt.Fatalf(\"IsLeapYear(%d) = %t, want %t\", tc.year, actual, tc.expected)\n\t\t\t}\n\t\t})\n\t}\n}",
+			"message": "\n=== RUN   TestLeapYears/year_divisible_by_100,_not_divisible_by_400_in_common_year\n\n    leap_test.go:10: IsLeapYear(2100) = true, want false\n\n--- FAIL: TestLeapYears/year_divisible_by_100,_not_divisible_by_400_in_common_year (0.00s)\n"
+		},
+		{
+			"name": "TestLeapYears/ year divisible by 100 but not by 3 is still not a leap year",
+			"status": "fail",
+			"test_code": "func TestLeapYears(t *testing.T) {\n\tfor _, tc := range testCases {\n\t\tt.Run(tc.description, func(t *testing.T) {\n\t\t\tactual := IsLeapYear(tc.year)\n\t\t\tif actual != tc.expected {\n\t\t\t\tt.Fatalf(\"IsLeapYear(%d) = %t, want %t\", tc.year, actual, tc.expected)\n\t\t\t}\n\t\t})\n\t}\n}",
+			"message": "\n=== RUN   TestLeapYears/year_divisible_by_100_but_not_by_3_is_still_not_a_leap_year\n\n    leap_test.go:10: IsLeapYear(1900) = true, want false\n\n--- FAIL: TestLeapYears/year_divisible_by_100_but_not_by_3_is_still_not_a_leap_year (0.00s)\n"
+		},
+		{
+			"name": "TestLeapYears/ year divisible by 400 is leap year",
+			"status": "fail",
+			"test_code": "func TestLeapYears(t *testing.T) {\n\tfor _, tc := range testCases {\n\t\tt.Run(tc.description, func(t *testing.T) {\n\t\t\tactual := IsLeapYear(tc.year)\n\t\t\tif actual != tc.expected {\n\t\t\t\tt.Fatalf(\"IsLeapYear(%d) = %t, want %t\", tc.year, actual, tc.expected)\n\t\t\t}\n\t\t})\n\t}\n}",
+			"message": "\n=== RUN   TestLeapYears/year_divisible_by_400_is_leap_year\n\n    leap_test.go:10: IsLeapYear(2000) = false, want true\n\n--- FAIL: TestLeapYears/year_divisible_by_400_is_leap_year (0.00s)\n"
+		},
+		{
+			"name": "TestLeapYears/ year divisible by 400 but not by 125 is still a leap year",
+			"status": "fail",
+			"test_code": "func TestLeapYears(t *testing.T) {\n\tfor _, tc := range testCases {\n\t\tt.Run(tc.description, func(t *testing.T) {\n\t\t\tactual := IsLeapYear(tc.year)\n\t\t\tif actual != tc.expected {\n\t\t\t\tt.Fatalf(\"IsLeapYear(%d) = %t, want %t\", tc.year, actual, tc.expected)\n\t\t\t}\n\t\t})\n\t}\n}",
+			"message": "\n=== RUN   TestLeapYears/year_divisible_by_400_but_not_by_125_is_still_a_leap_year\n\n    leap_test.go:10: IsLeapYear(2400) = false, want true\n\n--- FAIL: TestLeapYears/year_divisible_by_400_but_not_by_125_is_still_a_leap_year (0.00s)\n"
+		},
+		{
+			"name": "TestLeapYears/ year divisible by 200, not divisible by 400 in common year",
+			"status": "fail",
+			"test_code": "func TestLeapYears(t *testing.T) {\n\tfor _, tc := range testCases {\n\t\tt.Run(tc.description, func(t *testing.T) {\n\t\t\tactual := IsLeapYear(tc.year)\n\t\t\tif actual != tc.expected {\n\t\t\t\tt.Fatalf(\"IsLeapYear(%d) = %t, want %t\", tc.year, actual, tc.expected)\n\t\t\t}\n\t\t})\n\t}\n}",
+			"message": "\n=== RUN   TestLeapYears/year_divisible_by_200,_not_divisible_by_400_in_common_year\n\n    leap_test.go:10: IsLeapYear(1800) = true, want false\n\n--- FAIL: TestLeapYears/year_divisible_by_200,_not_divisible_by_400_in_common_year (0.00s)\n"
+		}
+	]
+}

tests/all-fail/go.mod

@@ -0,0 +1,3 @@
+module leap
+
+go 1.18

tests/all-fail/leap.go

@@ -0,0 +1,5 @@
+package leap
+
+func IsLeapYear(i int) bool {
+	return !(i%4 == 0 && i%100 != 0 || i%400 == 0)
+}

tests/all-fail/leap_test.go

@@ -0,0 +1,26 @@
+package leap
+
+import "testing"
+
+func TestLeapYears(t *testing.T) {
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			actual := IsLeapYear(tc.year)
+			if actual != tc.expected {
+				t.Fatalf("IsLeapYear(%d) = %t, want %t", tc.year, actual, tc.expected)
+			}
+		})
+	}
+}
+
+// Benchmark 400 years interval to get fair weighting of different years.
+func Benchmark400(b *testing.B) {
+	if testing.Short() {
+		b.Skip("skipping benchmark in short mode.")
+	}
+	for i := 0; i < b.N; i++ {
+		for y := 1600; y < 2000; y++ {
+			IsLeapYear(y)
+		}
+	}
+}