diff --git a/.evergreen/config.yml b/.evergreen/config.yml index ca2491ad6..b0264f1da 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -122,6 +122,7 @@ functions: script: | ${PREPARE_SHELL} sh ${DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh + "upload-mo-artifacts": - command: shell.exec params: @@ -152,7 +153,12 @@ functions: SSL=${SSL} \ AUTH=${AUTH} \ SWIFT_VERSION=${SWIFT_VERSION} \ - sh ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh + ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh + + "upload test results": + - command: attach.xunit_results + params: + file: ./src/testResults.xml "run atlas tests": - command: shell.exec @@ -199,6 +205,7 @@ pre: post: - func: "stop mongo-orchestration" - func: "upload-mo-artifacts" + - func: "upload test results" - func: "cleanup" tasks: diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh old mode 100644 new mode 100755 index d3094412f..7c06c9eff --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -1,15 +1,17 @@ -#!/bin/sh +#!/bin/bash set -o xtrace # Write all commands first to stderr set -o errexit # Exit the script with error if any of the commands fail # variables PROJECT_DIRECTORY=${PROJECT_DIRECTORY:-$PWD} MONGODB_URI=${MONGODB_URI:-"NO_URI_PROVIDED"} -SWIFT_VERSION=${SWIFT_VERSION:-5.0.3} +SWIFT_VERSION=${SWIFT_VERSION:-5.1} INSTALL_DIR="${PROJECT_DIRECTORY}/opt" TOPOLOGY=${TOPOLOGY:-single} OS=$(uname -s | tr '[:upper:]' '[:lower:]') EXTRA_FLAGS="-Xlinker -rpath -Xlinker ${INSTALL_DIR}/lib" +RAW_TEST_RESULTS="${PROJECT_DIRECTORY}/rawTestResults" +XML_TEST_RESULTS="${PROJECT_DIRECTORY}/testResults.xml" # ssl setup SSL=${SSL:-nossl} @@ -35,4 +37,16 @@ swiftenv local $SWIFT_VERSION swift build $EXTRA_FLAGS # test the driver -MONGODB_TOPOLOGY=${TOPOLOGY} MONGODB_URI=$MONGODB_URI swift test $EXTRA_FLAGS +set +o errexit # even if tests fail we want to parse the results, so disable errexit +set -o pipefail # propagate error codes in the following pipes + +MONGODB_TOPOLOGY=${TOPOLOGY} MONGODB_URI=$MONGODB_URI swift test $EXTRA_FLAGS 2>&1 | tee ${RAW_TEST_RESULTS} + +# save tests exit code +EXIT_CODE=$? + +# convert tests to XML +cat ${RAW_TEST_RESULTS} | swift "${PROJECT_DIRECTORY}/etc/convert-test-results.swift" > ${XML_TEST_RESULTS} + +# exit with exit code for running the tests +exit $EXIT_CODE diff --git a/.swiftlint.yml b/.swiftlint.yml index 1a4237c06..f6ba844f7 100644 --- a/.swiftlint.yml +++ b/.swiftlint.yml @@ -54,7 +54,6 @@ excluded: - Tests/LinuxMain.swift - Sources/MongoSwiftSync/Exports.swift - Sources/MongoSwift/MongoSwiftVersion.swift - - etc trailing_whitespace: ignores_comments: false diff --git a/etc/convert-test-results.swift b/etc/convert-test-results.swift new file mode 100644 index 000000000..814d173d5 --- /dev/null +++ b/etc/convert-test-results.swift @@ -0,0 +1,347 @@ +import Foundation + +// swiftlint:disable explicit_acl + +/// Represents a test suite. +struct TestSuite { + /// Name of the test suite. + let name: String + + /// Total execution time for the test suite, in seconds. + let time: TimeInterval + + /// Tests in the suite. + let tests: [TestCase] + + /// Count of tests in the suite. + let count: Int + + /// Count of failed tests in the suite. + let failureCount: Int + + /// Converts this test suite to XML. + func toXML() -> String { + var output = + """ + \n + """ + + for test in self.tests { + output += test.toXML() + } + + output += "\n" + return output + } +} + +/// Represents a test case. +struct TestCase { + /// The name of the class this test case belongs to. + let className: String + + /// The name of this test case. + let name: String + + /// The time the test case took to run, in seconds. + let time: TimeInterval + + /// Failure message produced by the test case, if any. + let failure: String? + + func toXML() -> String { + var output = + """ + \n + """ + + if let failure = self.failure { + // hack to replace disallowed XML characters with very similar unicode ones. + // evergreen doesn't render the escaped XML characters correctly so this preserves + // readability while keeping the XML valid. + let escapedFailure = failure + .replacingOccurrences(of: "\"", with: """) + .replacingOccurrences(of: "'", with: "'") + .replacingOccurrences(of: "<", with: "﹤") + .replacingOccurrences(of: ">", with: "﹥") + .replacingOccurrences(of: "&", with: "﹠") + + output += + """ + \n + """ + } + + output += "\n" + return output + } +} + +// Top-level redundant suites that we don't need to put into the xunit output. +let ignoreSuites = [ + // macOS + "AllTests", + "mongo-swift-driverPackageTests.xctest", + // linux + "All tests", + "debug.xctest" +] + +/// An error thrown while parsing test output. +struct ParsingError: LocalizedError { + let message: String + + public var errorDescription: String? { self.message } + + init(_ message: String) { + self.message = message + } +} + +extension NSTextCheckingResult { + func readMatch(at position: Int, in line: String) throws -> String { + guard let range = Range(self.range(at: position), in: line) else { + throw ParsingError("No capture group match at position \(position)") + } + return String(line[range]) + } +} + +extension TimeInterval { + init(input: String) throws { + guard let time = TimeInterval(input) else { + throw ParsingError("unable to parse TimeInterval from \(input)") + } + self = time + } +} + +func ensureSuiteMatches(old: String, new: String) throws { + guard old == new else { + throw ParsingError("test suite name \(new) does not match previously found name for current suite \(old)") + } +} + +/// State machine which processes `swift test` output and updates itself accordingly. +enum ParsingState { + /// In the following cases: + /// - `completeTests` is stored whenever we are in the middle of a suite, and contains any test cases we have found + /// for the suite so far. + /// - `completeSuites` is always stored and contains all of the suites we have fully parsed so far. + + /// We have read a line indicating that a test suite with the given name started, but we are not in a particular + /// test case. + case inSuite(name: String, completeTests: [TestCase], completeSuites: [TestSuite]) + /// We've read a line indicating that a suite with the given name ended, and are expecting to read a next line + /// containing test pass/fail counts for the suite. + case awaitingSuiteDetails(name: String, completeTests: [TestCase], completeSuites: [TestSuite]) + /// We are in the middle of a test with the given name in the given suite. `output` contains any output the test + /// case has produced. + case inTest(suite: String, name: String, output: [String], completeTests: [TestCase], completeSuites: [TestSuite]) + /// None of the above. This means we are between test suites. + case none(completeSuites: [TestSuite]) + +// account for variation in formatting of test output on platforms. this assumes you are running this script on the +// same platform where you ran the tests. disable force_try since we know these are valid regexes. +// swiftlint:disable force_try +#if os(macOS) + static let testCaseStartedRegex = try! NSRegularExpression(pattern: #"Test Case '-\[.+\.(.+) (.+)\]' started"#) + static let testCaseStatusRegex = try! NSRegularExpression( + pattern: #"Test Case '-\[.+\.(.+) (.+)\]' (passed|failed) \((.+) seconds\)"# + ) +#else + static let testCaseStartedRegex = try! NSRegularExpression(pattern: #"Test Case '(.+)\.(.+)' started"#) + static let testCaseStatusRegex = try! NSRegularExpression( + pattern: #"Test Case '(.+)\.(.+)' (passed|failed) \((.+) seconds\)"# + ) +#endif + static let testSuiteStartedRegex = try! NSRegularExpression(pattern: #"Test Suite '(.+)' started"#) + static let testSuiteStatusRegex = try! NSRegularExpression(pattern: #"Test Suite '(.+)' (passed|failed)"#) + static let testSuiteDetailsRegex = try! NSRegularExpression( + pattern: #"Executed (\d+) tests?, with (\d+) failures? \((\d+) unexpected\) in (.+) \("# + ) + // swiftlint:enable force_try + + /// Processes a new line of test output and updates self accordingly. + mutating func processLine(_ line: String) throws { + let fullRange = NSRange(line.startIndex.. + + """ + + completeSuites.map { $0.toXML() }.reduce("", +) + + "" + +print(fullXML)