[1.1.0]: Various fixes

bin/run.sh

@@ -31,7 +31,9 @@ results_file="${OUTPUT_DIR}/results.json"
 BASEDIR=$(dirname "$0")
 
 touch "${results_file}"
+
 export RUNALL=true
-swift test --package-path "${INPUT_DIR}" -v --parallel --xunit-output "${junit_file}" &> "${capture_file}"
+swift test --package-path "${INPUT_DIR}" -v --parallel --num-workers 1 --xunit-output "${junit_file}" &> "${capture_file}"
+
 
-./bin/TestRunner "${spec_file}" "${junit_file}" "${capture_file}" "${results_file}"
+./bin/TestRunner "${spec_file}" "${junit_file}" "${capture_file}" "${results_file}" "${SLUG}"

changelog.md

@@ -0,0 +1,26 @@
+# 1.1.0
+
+- The test runner will run one test at a time, this is to be able to give a better error message when a test fails.
+  - This may change in the future.
+- Improved message handling for unimplemented tests.
+- Improved message handling for non existing functions.
+- Fixed so the test runner will give messaging for when test doesn't use `XCTAssertEqual`.
+- Fixed so the test runner don't remove incorrect characters from test_code.
+  - This was caused when a test runner had an if statement, then would the closing bracket be removed.
+- Fixed so the test runner can now handle multiline assert statements (window system exercise).
+- Fixed so the test runner will no longer output if an error occurred when running the test. 
+- The test code will now be indented.
+- Slight changes in formatting of the test runners source code.
+
+# 1.0.1
+
+- Fixed an environment variable with caused so only the first test was run.
+
+# 1.0.0
+
+- Initial release
+- Add support for Swift 5.8
+- Test code
+- Task id
+- Fixes and performance improvements
+- Added so test will run in parallel

src/testrunner/Sources/TestRunner/main.swift

@@ -27,16 +27,21 @@ class TestRunner{
         var tests: [testCases] = []
         var taskId = 0
         var inClass = 0
-        let charactersToRemove = CharacterSet(charactersIn: "\n{} ")
+        let charactersToRemove = CharacterSet(charactersIn: "\n ")
 
         override func visit(_ node: FunctionDeclSyntax) -> SyntaxVisitorContinueKind {
             let bodyCheck = node.body
             if bodyCheck != nil{
                 let name: String = String(describing: node.identifier)
                 var body: String = String(describing: bodyCheck!).trimmingCharacters(in: charactersToRemove)
+                body = String(body[body.index(after: body.startIndex)..<body.index(before: body.endIndex)]).trimmingCharacters(in: charactersToRemove)
                 var bodyList = body.components(separatedBy: "\n")
-                for (rowIdx, row) in bodyList.enumerated(){
-                    bodyList[rowIdx] = row.trimmingCharacters(in: CharacterSet.whitespaces)
+                var count = 0
+                if bodyList.count > 1 {
+                    count = bodyList[1].prefix(while: { $0.isWhitespace }).count
+                }
+                for (rowIdx, row) in bodyList[1...].enumerated(){
+                    bodyList[rowIdx + 1] = String(row.dropFirst(count))
                 }
                 body = bodyList.joined(separator: "\n")
                 tests.append(testCases(name: name, test_code: body, task_id: taskId == 0 ? nil : taskId))
@@ -77,7 +82,7 @@ class TestRunner{
     }
 
 
-    func run(filePath : String, xmlPath : String, contextPath : String, resultPath : String){
+    func run(filePath : String, xmlPath : String, contextPath : String, resultPath : String, slug : String){
     var swiftSource = ""
     do {
         swiftSource = try NSString(contentsOfFile: filePath,
@@ -95,7 +100,7 @@ class TestRunner{
             encoding: String.Encoding.ascii.rawValue) as String
     }catch {
             print("Error xmlPath: \(error)")
-            writeJson(resultPath: resultPath, error: errorContext(context: contextPath))
+            writeJson(resultPath: resultPath, error: errorContext(context: contextPath, slug: slug))
             return
     }
 
@@ -113,23 +118,33 @@ class TestRunner{
     }catch {
             print("Error contextPath: \(error)")
         }
-    addContext(context: context)
+    addContext(context: context, slug: slug)
 
     writeJson(resultPath: resultPath)
     }
 
-    func addContext(context: String) {
+    func addContext(context: String, slug: String) {
         for (testIdx, testCase) in xmlTests.enumerated(){
             var error = false
             if testCase.status == "fail"{
                 let contextLines = context.components(separatedBy: "\n")[1...]
                 var found = false
-                for row in contextLines{
-                    if let startRange = row.range(of: "\(testCase.name) : XCTAssertEqual failed: ") {
-                        found = true
-                        if let endRange = row.range(of: "Test Case ", options: [], range: (startRange.upperBound..<row.endIndex)){
-                            let sum = row.distance(from: row.startIndex, to: startRange.upperBound)
-                            let lowerBound1 = String.Index(encodedOffset: sum + row.distance(from: startRange.upperBound, to: endRange.lowerBound))
+                var start = ""
+                var current = 0
+                for (rowIdx, row) in contextLines.enumerated(){
+                    if found {
+                        if let endRange = row.range(of: "Test Case "){
+                            let middle = contextLines[current..<rowIdx].joined(separator: "\n")
+                            let newSubstring = String(row[row.startIndex..<endRange.lowerBound])
+                            xmlTests[testIdx].message = start + "\n" + middle + newSubstring
+                            break
+                        }
+                    }
+                    else if let startRange = row.range(of: "\(testCase.name) : XCTAssert"){
+                            found = true
+                        if let endRange = row.range(of: "Test Case ", options: [], range: (startRange.lowerBound..<row.endIndex)){
+                            let sum = row.distance(from: row.startIndex, to: startRange.lowerBound)
+                            let lowerBound1 = String.Index(encodedOffset: sum + row.distance(from: startRange.lowerBound, to: endRange.lowerBound))
                             let upperBound1 = String.Index(encodedOffset: sum)
                             let newSubstring = row[upperBound1..<lowerBound1]
                             xmlTests[testIdx].message = String(newSubstring)
@@ -138,8 +153,9 @@ class TestRunner{
                             let sum = row.distance(from: row.startIndex, to: startRange.upperBound)
                             let upperBound1 = String.Index(encodedOffset: sum)
                             let newSubstring = row[upperBound1..<row.endIndex]
-                            xmlTests[testIdx].message = String(newSubstring)
-                            break
+                            start = String(newSubstring)
+                            current = rowIdx + 2
+                            continue
                         }
                     }
                 }
@@ -154,57 +170,68 @@ class TestRunner{
                                 current = testCase.name
                                 if let startRange = row.range(of: "started at", options: .backwards) {
                                     if let newStartRange = row.range(of: ".", options: [], range: (startRange.upperBound..<row.endIndex)){
+                                        let newIndex = row.index(newStartRange.upperBound, offsetBy: 3)
+                                        let sum = row.distance(from: row.startIndex, to: newIndex)
                                         if let endRange = row.range(of: "Test Case ", options: [], range: (newStartRange.upperBound..<row.endIndex)){
-                                            let newIndex = row.index(newStartRange.upperBound, offsetBy: 3)
-                                            let sum = row.distance(from: row.startIndex, to: newIndex)
                                             let lowerBound1 = String.Index(encodedOffset: sum + row.distance(from: newIndex, to: endRange.lowerBound))
                                             let upperBound1 = String.Index(encodedOffset: sum)
                                             let newSubstring = row[upperBound1..<lowerBound1]
                                             xmlTests[testIdx].message = String(newSubstring)
                                             break
+                                        } else if let endRange = row.range(of: "Current stack trace:", options: [], range: (newStartRange.upperBound..<row.endIndex)){
+                                            let lowerBound1 = String.Index(encodedOffset: sum + row.distance(from: newIndex, to: endRange.lowerBound))
+                                            let upperBound1 = String.Index(encodedOffset: sum)
+                                            let newSubstring = row[upperBound1..<lowerBound1]
+                                            xmlTests[testIdx].message = String(newSubstring)
+                                            break
+                                        } else {
+                                            let lowerBound1 = String.Index(encodedOffset: sum)
+                                            let newSubstring = row[lowerBound1..<row.endIndex]
+                                            xmlTests[testIdx].message = String(newSubstring)
+                                            break
                                         }
                                     }
                                 }
-                        }   
+                            }
+                        }
                     }
                 }
             }
-        }
-        if !error {
-            let contextLinesOutput = context.components(separatedBy: "\n")[1...]
-            var start = 0
-            var startString = ""
-            for (rowIdx, row) in contextLinesOutput.enumerated(){
-                let startIndex = row.index(row.startIndex, offsetBy: 0)
-                if row.count > 0{
-                    if start == 0 && row.contains(testCase.name + "'") && row[startIndex] != "["{
-                        if let startRange = row.range(of: "started at", options: .backwards) {
-                            if let newStartRange = row.range(of: ".", options: [], range: (startRange.upperBound..<row.endIndex)){
-                                let newIndex = row.index(newStartRange.upperBound, offsetBy: 3)
-                                startString = String(row[newIndex..<row.endIndex])
-                                if startString.hasPrefix("Test Case") || startString.contains(testCase.name + "'"){
-                                    startString = ""
-                                    break
+            if !error {
+                let contextLinesOutput = context.components(separatedBy: "\n")[1...]
+                var start = 0
+                var startString = ""
+                for (rowIdx, row) in contextLinesOutput.enumerated(){
+                    let startIndex = row.index(row.startIndex, offsetBy: 0)
+                    if row.count > 0{
+                        if start == 0 && row.contains(testCase.name + "'") && row[startIndex] != "[" && !row.contains("error"){
+                            if let startRange = row.range(of: "started at", options: .backwards) {
+                                if let newStartRange = row.range(of: ".", options: [], range: (startRange.upperBound..<row.endIndex)){
+                                    let newIndex = row.index(newStartRange.upperBound, offsetBy: 3)
+                                    startString = String(row[newIndex..<row.endIndex])
+                                    if startString.hasPrefix("Test Case") || startString.contains(testCase.name + "'"){
+                                        startString = ""
+                                        break
+                                    }
                                 }
                             }
-                        }
-                        start = rowIdx + 2
-                    } else if (row.contains(testCase.name + "'") && row[startIndex] != "[") || row.hasPrefix("\\")  {
-                        if start < rowIdx{
-                            xmlTests[testIdx].output = startString + "\n" + contextLinesOutput[start...rowIdx].joined(separator: "\n")
-                        }
-                        break
-                    }   
+                            start = rowIdx + 2
+                        } else if (row.contains(testCase.name + "'") && row[startIndex] != "[" && !row.contains("error")) || row.hasPrefix("\\") {
+                            if start < rowIdx{
+                                xmlTests[testIdx].output = startString + "\n" + contextLinesOutput[start...rowIdx].joined(separator: "\n")
+                            }
+                            break
+                        }   
+                    }
+                }
+                if startString != "" && xmlTests[testIdx].output == nil{
+                    xmlTests[testIdx].output = startString
                 }
-            }
-            if startString != "" && xmlTests[testIdx].output == nil{
-                xmlTests[testIdx].output = startString
             }
         }
     }
-    }
 
-    func errorContext(context : String) -> String{
+    func errorContext(context : String, slug : String) -> String{
         var something = ""
         do {
             // Use contentsOfFile overload.
@@ -220,7 +247,7 @@ class TestRunner{
         let message = something.components(separatedBy: "\n")[1...]
         var start = 0
         for (rowIdx, row) in message.enumerated(){
-            if row.contains("CompileError.swift:"){
+            if row.contains("CompileError.swift:") || row.contains("\(camelCase(slug: slug))Tests.swift:"){
                 start = rowIdx + 1
                 break
             }
@@ -261,6 +288,17 @@ class TestRunner{
             return
         }
     }
+
+    private func camelCase(slug: String) -> String{
+        let words = slug.components(separatedBy: "-")
+        var camelCase = ""
+        for word in words{
+            if word != ""{
+                camelCase += word.prefix(1).uppercased() + word.dropFirst()
+            }
+        }
+        return camelCase
+    }
 }
 
-TestRunner().run(filePath: CommandLine.arguments[1], xmlPath: CommandLine.arguments[2], contextPath:  CommandLine.arguments[3], resultPath: CommandLine.arguments[4])
+TestRunner().run(filePath: CommandLine.arguments[1], xmlPath: CommandLine.arguments[2], contextPath:  CommandLine.arguments[3], resultPath: CommandLine.arguments[4], slug : CommandLine.arguments[5])

tests/multiple-tests-multiple-fails/expected_results.json

@@ -2,14 +2,14 @@
   "status" : "fail",
   "tests" : [
     {
-      "message" : "(\"-1\") is not equal to (\"5\") - 2+3 should equal 5",
+      "message" : "testAdd : XCTAssertEqual failed: (\"-1\") is not equal to (\"5\") - 2+3 should equal 5",
       "name" : "testAdd",
       "status" : "fail",
       "task_id" : 1,
       "test_code" : "XCTAssertEqual(sum(2, 3), 5, \"2+3 should equal 5\")"
     },
     {
-      "message" : "(\"5\") is not equal to (\"-1\") -",
+      "message" : "testSub : XCTAssertEqual failed: (\"5\") is not equal to (\"-1\") -",
       "name" : "testSub",
       "status" : "fail",
       "task_id" : 1,

tests/multiple-tests-single-fail/expected_results.json

@@ -2,7 +2,7 @@
   "status" : "fail",
   "tests" : [
     {
-      "message" : "(\"0\") is not equal to (\"5\") - 2+3 should equal 5",
+      "message" : "testAdd : XCTAssertEqual failed: (\"0\") is not equal to (\"5\") - 2+3 should equal 5",
       "name" : "testAdd",
       "output" : "yabba",
       "status" : "fail",
@@ -24,7 +24,7 @@
       "test_code" : "XCTAssertEqual(mul(2, 3), 6)"
     },
     {
-      "message" : "(\"0\") is not equal to (\"25\") - 12+13 should equal 25",
+      "message" : "testAdd_2 : XCTAssertEqual failed: (\"0\") is not equal to (\"25\") - 12+13 should equal 25",
       "name" : "testAdd_2",
       "output" : "yabba",
       "status" : "fail",

tests/multiple-tests-with-exception/expected_results.json

@@ -17,7 +17,7 @@
       "test_code" : "XCTAssertEqual(mul(3, 2), 6)"
     },
     {
-      "message" : "/solution/Tests/MultipleWithExceptionTests/MultipleWithExceptionTests.swift:19: error: MultipleWithExceptionTests.testThrow : XCTAssertEqual threw error \"testError(\"Oh noes! Div by zeroes!!!\")\" -",
+      "message" : "testThrow : XCTAssertEqual threw error \"testError(\"Oh noes! Div by zeroes!!!\")\" -",
       "name" : "testThrow",
       "status" : "fail",
       "test_code" : "XCTAssertEqual(try throwErr(2, 0), 6)"

tests/single-test-that-fails/expected_results.json

@@ -2,7 +2,7 @@
   "status" : "fail",
   "tests" : [
     {
-      "message" : "(\"-1\") is not equal to (\"5\") - 2+3 should equal 5",
+      "message" : "testAdd : XCTAssertEqual failed: (\"-1\") is not equal to (\"5\") - 2+3 should equal 5",
       "name" : "testAdd",
       "status" : "fail",
       "test_code" : "XCTAssertEqual(sum(2, 3), 5, \"2+3 should equal 5\")"

tests/single-test-with-exception/expected_results.json

@@ -2,7 +2,7 @@
   "status" : "fail",
   "tests" : [
     {
-      "message" : "/solution/Tests/SingleWithExceptionTests/SingleWithExceptionTests.swift:7: error: SingleWithExceptionTests.testAdd : XCTAssertEqual threw error \"testError(\"Kaboomtown!\")\" - 2+3 should equal 5",
+      "message" : "testAdd : XCTAssertEqual threw error \"testError(\"Kaboomtown!\")\" - 2+3 should equal 5",
       "name" : "testAdd",
       "status" : "fail",
       "test_code" : "XCTAssertEqual(try sum(2, 3), 5, \"2+3 should equal 5\")"