Fixes #50 (#51)

- do not rely on exercise slug or .config
- scan entire test output folder for "TEST-*.xml" files

Dockerfile

@@ -2,8 +2,6 @@ FROM sbtscala/scala-sbt:openjdk-8u342_1.7.3_2.13.10
 
 WORKDIR /opt/test-runner
 
-ENV CONFIG_PATH '.meta'
-
 RUN apt-get update
 RUN apt-get install --yes jq
 

bin/run-in-docker.sh

@@ -41,5 +41,4 @@ docker run \
     --mount type=bind,src="${input_dir}",dst=/solution \
     --mount type=bind,src="${output_dir}",dst=/output \
     --mount type=tmpfs,dst=/tmp \
-    --env CONFIG_PATH="${CONFIG_PATH}" \
-    exercism/test-runner "${slug}" /solution /output 
+    exercism/test-runner "${slug}" /solution /output

bin/run-tests.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env sh
+#!/usr/bin/env bash
 
 # Synopsis:
 # Test the test runner by running it against a predefined set of solutions 
@@ -30,7 +30,7 @@ for test_dir in tests/*; do
       "${results_file_path}"
 
     echo "${test_dir_name}: comparing results.json to expected_results.json"
-    diff "${results_file_path}" "${expected_results_file_path}"
+    diff <(jq < "${results_file_path}") <(jq < "${expected_results_file_path}")
 
     if [ $? -ne 0 ]; then
         exit_code=1

bin/run.sh

@@ -24,33 +24,17 @@ fi
 slug="$1"
 input_dir="${2%/}"
 output_dir="${3%/}"
-path=${CONFIG_PATH:-'.meta'}
 
 test_runner_jar=/opt/test-runner/target/scala-2.13/TestRunner-assembly-0.1.0-SNAPSHOT.jar
 
 workdir=/tmp/exercise
-workdir_target=/tmp/exercise/target
+workdir_target="${workdir}/target"
+tests_reports_folder="${workdir}/test-reports"
 
 results_file="${output_dir}/results.json"
 build_log_file="${output_dir}/build.log"
 runner_log_file="${output_dir}/runner.log"
 
-function test_name_from_config() {
-    config="${input_dir}/${path}/config.json"
-    [[ -f "${config}" ]] || return
-    test_files=($(cat "${config}" | jq -r '.files.test[]'))
-    [[ "${#test_files[@]}" == '1' ]] || return
-    test_file=$(basename "${test_files[0]}")
-    echo "${test_file/.scala/}"
-}
-
-function test_name_from_slug() {
-  exercise=$(echo "${slug}" | sed -E 's/(^|-)([a-z])/\U\2/g')
-  echo "${exercise}Test"
-}
-
-tests_results_file="${workdir}/test-reports/TEST-$(test_name_from_config || test_name_from_slug).xml"
-
 # Create the output directory if it doesn't exist
 mkdir -p "${output_dir}"
 
@@ -74,7 +58,7 @@ scalac -classpath "${test_runner_jar}" -d "${workdir_target}" "${workdir}"/src/m
 scala -classpath "${test_runner_jar}" org.scalatest.tools.Runner -R "${workdir_target}" -u "${workdir}"/test-reports
 
 # Write the results.json file in the exercism format
-java -jar "${test_runner_jar}" "${build_log_file}" "${tests_results_file}" "${results_file}" &> "${runner_log_file}"
+java -jar "${test_runner_jar}" "${build_log_file}" "${tests_reports_folder}" "${results_file}" &> "${runner_log_file}"
 
 # change workdir back to the original input_dir in the final results file
 sed -i "s~${workdir}~${input_dir}~g" "${results_file}"

src/main/scala/Application.scala

@@ -1,74 +1,80 @@
-  import scala.io.Source
-  import org.json.{JSONArray, JSONObject, XML}
+import org.json.{JSONArray, JSONObject, XML}
 
-  import java.io.{File, FileWriter}
+import java.io.{File, FileFilter, FileWriter}
+import scala.io.Source
 
-  object Application extends App {
-    require(args.length == 3, "Invalid number of arguments. Expected: <build-log-file-path> <test-results-file-path> <results-json-file-path>")
-    val buildLogFilePath = args(0)
-    val testResultsFilePath = args(1)
-    val resultsJsonFilePath = args(2)
+object Application extends App {
+  require(args.length == 3, s"Invalid number of arguments. Expected: <build-log-file-path> <test-results-folder-path> <results-json-file-path>, got: ${args.mkString("", ", ", "")}")
+  val buildLogFilePath = args(0)
+  val testResultsFolderPath = args(1)
+  val resultsJsonFilePath = args(2)
 
-    writeResultsJSON(buildLogFilePath, testResultsFilePath, resultsJsonFilePath)
-
-    def writeResultsJSON(buildLogFilePath: String, testResultsFilePath: String, resultsJsonFilePath: String): Unit = {
-      val resultsJsonFile = new File(resultsJsonFilePath)
-      val resultsJsonFileWriter = new FileWriter(resultsJsonFile)
+  val testResultsFolder = new File(testResultsFolderPath)
+  if (!testResultsFolder.isDirectory) {
+    throw new RuntimeException(s"Expected $testResultsFolderPath to be a folder")
+  }
+  val testResultFiles = testResultsFolder.listFiles(new FileFilter() {
+    override def accept(file: File): Boolean = file.getName.matches("TEST-.*\\.xml")
+  }).toList
+  writeResultsJSON(buildLogFilePath, testResultFiles, resultsJsonFilePath)
 
-      val json = toExercismJSON(buildLogFilePath, testResultsFilePath)
-      json.write(resultsJsonFileWriter)
-      resultsJsonFileWriter.close()
-    }
+  def writeResultsJSON(buildLogFilePath: String, testResultsFiles: List[File], resultsJsonFilePath: String): Unit = {
+    val resultsJsonFile = new File(resultsJsonFilePath)
+    val resultsJsonFileWriter = new FileWriter(resultsJsonFile)
 
-    def getTestSuiteObject(testResultsFilePath: String): JSONObject = {
-      val bufferedSource = Source.fromFile(testResultsFilePath)
-      val xml = bufferedSource.mkString
-      bufferedSource.close
-      XML.toJSONObject(xml).getJSONObject("testsuite")
-    }
+    val json = toExercismJSON(buildLogFilePath, testResultsFiles)
+    json.write(resultsJsonFileWriter)
+    resultsJsonFileWriter.close()
+  }
 
-    def getTestCasesJSON(testResultsFilePath: String): JSONArray = {
-      getTestSuiteObject(testResultsFilePath).getJSONArray("testcase")
-    }
+  def getTestSuiteObject(testResultsFile: File): JSONObject = {
+    val bufferedSource = Source.fromFile(testResultsFile)
+    val xml = bufferedSource.mkString
+    bufferedSource.close
+    XML.toJSONObject(xml).getJSONObject("testsuite")
+  }
 
-    // log, not xml
-    def findErrorsInLog(buildLogFilePath: String): String = {
-      val fileSource = Source.fromFile(buildLogFilePath)
-      val rawContent = fileSource.mkString
-      fileSource.close
-      if (rawContent.contains("error: ")) rawContent else ""
-    }
+  // log, not xml
+  def findErrorsInLog(buildLogFilePath: String): String = {
+    val fileSource = Source.fromFile(buildLogFilePath)
+    val rawContent = fileSource.mkString
+    fileSource.close
+    if (rawContent.contains("error: ")) rawContent else ""
+  }
 
-    def toTestCaseJSON(testCase: JSONObject): JSONObject = {
-      val fail = testCase.optJSONObject("failure")
-      new JSONObject()
+  def toTestCaseJSON(testCase: JSONObject): JSONObject = {
+    val fail = testCase.optJSONObject("failure")
+    new JSONObject()
       .put("name", testCase.get("name").toString)
-      .put("status", if(fail != null) "fail" else "pass" )
-      .put("message", if(fail != null) fail.getString("message") else JSONObject.NULL)
+      .put("status", if (fail != null) "fail" else "pass")
+      .put("message", if (fail != null) fail.getString("message") else JSONObject.NULL)
       .put("output", JSONObject.NULL)
       .put("test_code", JSONObject.NULL)
-    }
+  }
 
-    def toExercismJSON(buildLogFilePath: String, testResultsFilePath: String): JSONObject = {
-      val baseObject = new JSONObject().put("version", 2)
-      val errorMessage = findErrorsInLog(buildLogFilePath)
-      if(errorMessage.nonEmpty) {
-        baseObject
+  def toExercismJSON(buildLogFilePath: String, testResultsFiles: List[File]): JSONObject = {
+    val baseObject = new JSONObject().put("version", 2)
+    val errorMessage = findErrorsInLog(buildLogFilePath)
+    if (errorMessage.nonEmpty) {
+      baseObject
         .put("status", "error")
         .put("message", errorMessage)
-      } else {
-        val testSuite = getTestSuiteObject(testResultsFilePath)
-        val failuresNum = testSuite.getInt("failures")
-        val testcase = testSuite.get("testcase")
-        val testCases: Array[JSONObject] = testcase match {
-          case arr: JSONArray => (0 until arr.length).map(idx => toTestCaseJSON(arr.getJSONObject(idx))).toArray
-          case obj: JSONObject => Array(toTestCaseJSON(obj))
-        }
-
-        baseObject
-        .put("status", if(failuresNum > 0) "fail" else "pass")
+    } else {
+      val (failuresCount, testCases) = testResultsFiles.map(getTestSuiteObject)
+        .filter(testSuite => testSuite.has("testcase"))
+        .map(testSuite => {
+          val failuresCount = testSuite.getInt("failures")
+          val testcase = testSuite.get("testcase")
+          val testCases: Array[JSONObject] = testcase match {
+            case arr: JSONArray => (0 until arr.length).map(idx => toTestCaseJSON(arr.getJSONObject(idx))).toArray
+            case obj: JSONObject => Array(toTestCaseJSON(obj))
+          }
+          (failuresCount, testCases)
+        }).reduce((a, b) => (a._1 + b._1, Array.concat(a._2, b._2)))
+      baseObject
+        .put("status", if (failuresCount > 0) "fail" else "pass")
         .put("message", JSONObject.NULL)
         .put("tests", testCases)
-      }
     }
   }
+}

src/test/scala/ApplicationSpec.scala

@@ -1,20 +1,27 @@
-import org.json.JSONObject
+import org.json.{JSONArray, JSONObject}
 import org.scalatest.funsuite.AnyFunSuite
 import org.scalatest.matchers.should.Matchers
 
+import java.io.File
+
 class ApplicationSpec extends AnyFunSuite with Matchers {
 
+  def getTestCasesJSON(path: String): JSONArray = {
+    val file = new File(path)
+    Application.getTestSuiteObject(file).getJSONArray("testcase")
+  }
+
   test("A successful xml should pass simply") {
     val xmlTestURL = getClass.getResource("/GradeSchool_successful.xml").getPath
-    val jsonArray = Application.getTestCasesJSON(xmlTestURL)
-    val objects = (0 until jsonArray.length).map( jsonArray.getJSONObject(_).optJSONObject("failure") )
-    objects should contain only (null)
+    val jsonArray = getTestCasesJSON(xmlTestURL)
+    val objects = (0 until jsonArray.length).map(jsonArray.getJSONObject(_).optJSONObject("failure"))
+    objects should contain only null
   }
 
   test("A successful xml should be properly formatted as JSON") {
     val xmlTestURL = getClass.getResource("/GradeSchool_successful.xml").getPath
     val outputFileURL = getClass.getResource("/outputs/output.txt").getPath
-    val exercismOutput: JSONObject = Application.toExercismJSON(outputFileURL, xmlTestURL)
+    val exercismOutput: JSONObject = Application.toExercismJSON(outputFileURL, List(new File(xmlTestURL)))
 
     assert(exercismOutput.getInt("version") == 2)
     assert(exercismOutput.getString("status") == "pass")
@@ -34,7 +41,7 @@ class ApplicationSpec extends AnyFunSuite with Matchers {
   test("A successful xml with a single test case should be properly formatted as JSON") {
     val xmlTestURL = getClass.getResource("/HelloWorld_successful.xml").getPath
     val outputFileURL = getClass.getResource("/outputs/output.txt").getPath
-    val exercismOutput: JSONObject = Application.toExercismJSON(outputFileURL, xmlTestURL)
+    val exercismOutput: JSONObject = Application.toExercismJSON(outputFileURL, List(new File(xmlTestURL)))
 
     assert(exercismOutput.getInt("version") == 2)
     assert(exercismOutput.getString("status") == "pass")
@@ -46,16 +53,16 @@ class ApplicationSpec extends AnyFunSuite with Matchers {
   }
 
   test("A failing xml should contain a failure object") {
-    val xmlTestURL = getClass.getResource("/GradeSchool_failure.xml").getPath
-    val jsonArray = Application.getTestCasesJSON(xmlTestURL)
-    val objects = (0 until jsonArray.length).map( jsonArray.getJSONObject(_).optJSONObject("failure") )
-    objects.filter( _ !== null ).length > 0
+    val xmlTestURL = getClass.getResource("/GradeSchool_failure.xml").getFile
+    val jsonArray = getTestCasesJSON(xmlTestURL)
+    val objects = (0 until jsonArray.length).map(jsonArray.getJSONObject(_).optJSONObject("failure"))
+    objects.exists(_ !== null)
   }
 
   test("A failing xml should be properly formatted as JSON") {
     val xmlTestURL = getClass.getResource("/GradeSchool_failure.xml").getPath
     val outputFileURL = getClass.getResource("/outputs/output_fail.txt").getPath
-    val exercismOutput: JSONObject = Application.toExercismJSON(outputFileURL, xmlTestURL)
+    val exercismOutput: JSONObject = Application.toExercismJSON(outputFileURL, List(new File(xmlTestURL)))
     assert(exercismOutput.getInt("version") == 2)
     assert(exercismOutput.getString("status") == "fail")
     assert(exercismOutput.opt("message") == null)

tests/example-multiple-tests/.meta/config.json

@@ -0,0 +1,9 @@
+{
+  "blurb": "Verify that expected test class names can be extracted from .meta/config.json",
+  "files": {
+    "test": [
+      "src/test/scala/ExampleMultipleFilesTest.scala",
+      "src/test/scala/ExampleMultipleClassesTest.scala"
+    ]
+  }
+}

tests/example-multiple-tests/build.sbt

@@ -0,0 +1,3 @@
+scalaVersion := "2.13.6"
+
+libraryDependencies += "org.scalatest" %% "scalatest" % "3.2.10" % "test"

tests/example-multiple-tests/expected_results.json

@@ -0,0 +1,28 @@
+{
+  "tests": [
+    {
+      "output": null,
+      "name": "two-one",
+      "test_code": null,
+      "message": null,
+      "status": "pass"
+    },
+    {
+      "output": null,
+      "name": "one-one",
+      "test_code": null,
+      "message": null,
+      "status": "pass"
+    },
+    {
+      "output": null,
+      "name": "one-two",
+      "test_code": null,
+      "message": null,
+      "status": "pass"
+    }
+  ],
+  "message": null,
+  "version": 2,
+  "status": "pass"
+}

tests/example-multiple-tests/src/test/scala/ExampleMulitpleFilesTest.scala

@@ -0,0 +1,8 @@
+import org.scalatest.funsuite.AnyFunSuite
+import org.scalatest.matchers.should.Matchers
+
+class ExampleMultipleFilesTest extends AnyFunSuite with Matchers {
+  test("two-one") {
+    true should be (true)
+  }
+}

tests/example-multiple-tests/src/test/scala/ExampleMultipleClassesTest.scala

@@ -0,0 +1,15 @@
+import org.scalatest.funsuite.AnyFunSuite
+import org.scalatest.matchers.should.Matchers
+
+/** @version 1.3.0 */
+class ExampleMultipleTestsTest extends AnyFunSuite with Matchers {
+  test("one-one") {
+    true should be (true)
+  }
+}
+
+class ExampleAnotherMultipleTestsTest extends AnyFunSuite with Matchers {
+  test("one-two") {
+    true should be (true)
+  }
+}