diff --git a/.swift-version b/.swift-version new file mode 100644 index 0000000..5186d07 --- /dev/null +++ b/.swift-version @@ -0,0 +1 @@ +4.0 diff --git a/Package.swift b/Package.swift index 5cf7199..6d29f7b 100644 --- a/Package.swift +++ b/Package.swift @@ -11,12 +11,12 @@ let package = Package( ], dependencies: [ .package(url: "https://github.com/onevcat/Rainbow.git", from: "3.0.0"), - .package(url: "https://github.com/kareman/SwiftShell.git", from: "4.0.0"), + .package(url: "https://github.com/harlanhaskins/ShellOut.git", .branch("on-your-mark-get-set-go")), ], targets: [ .target( name: "LiteSupport", - dependencies: ["Rainbow", "SwiftShell"]), + dependencies: ["Rainbow", "ShellOut"]), // This needs to be named `lite-test` instead of `lite` because consumers // of `lite` should use the target name `lite`. diff --git a/README.md b/README.md index 5e993d5..cd41b72 100644 --- a/README.md +++ b/README.md @@ -24,10 +24,10 @@ depends on the Lite support library, `LiteSupport`. ### Making a `lite` Target From that target's `main.swift`, make a call to -`runLite(substitutions:pathExtensions:testDirPath:testLinePrefix:)`. This call +`runLite(substitutions:pathExtensions:testDirPath:testLinePrefix:parallelismLevel:)`. This call is the main entry point to `lite`'s test running. -It takes 4 arguments: +It takes 5 arguments: | Argument | Description | |----------|-------------| @@ -35,6 +35,7 @@ It takes 4 arguments: | `pathExtensions` | The set of path extensions that Lite should search for when discovering tests. | | `testDirPath` | The directory in which Lite should look for tests. Lite will perform a deep search through this directory for all files whose extension exists in `pathExtensions` and which have valid RUN lines. | | `testLinePrefix` | The prefix before `RUN:` in a file. This is almost always your specific langauge's line comment syntax. | +| `parallelismLevel` | Specifies the amount of parallelism to apply to the test running process. Default value is `.none`, but you can provide `.automatic` to use the available machine cores, or `.explicit(n)` to specify an explicit number of parallel tests | > Note: An example consumer of `Lite` exists in this repository as `lite-test`. diff --git a/Sources/LiteSupport/ParallelExecutor.swift b/Sources/LiteSupport/ParallelExecutor.swift new file mode 100644 index 0000000..0eab4b4 --- /dev/null +++ b/Sources/LiteSupport/ParallelExecutor.swift @@ -0,0 +1,74 @@ +/// ParallelExecutor.swift +/// +/// Copyright 2017, The Silt Language Project. +/// +/// This project is released under the MIT license, a copy of which is +/// available in the repository. + +import Foundation +import Dispatch + +/// A class that handles executing tasks in a round-robin fashion among +/// a fixed number of workers. It uses GCD to split the work among a fixed +/// set of queues and automatically manages balancing workloads between workers. +final class ParallelExecutor { + /// The set of worker queues on which to add tasks. + private let queues: [DispatchQueue] + + /// The dispatch group on which to synchronize the workers. + private let group = DispatchGroup() + + /// The results from each task executed on the workers, in non-deterministic + /// order. + private var results = [TaskResult]() + + /// The queue on which to protect the results array. + private let resultQueue = DispatchQueue(label: "parallel-results") + + /// The current number of tasks, used for round-robin dispatch. + private var taskCount = 0 + + /// Creates an executor that splits tasks among the provided number of + /// workers. + /// - parameter numberOfWorkers: The number of workers to spawn. This number + /// should be <= the number of hyperthreaded + /// cores on your machine, to avoid excessive + /// context switching. + init(numberOfWorkers: Int) { + self.queues = (0.. TaskResult) { + queues[nextTask % queues.count].async(group: group) { + self.addResult(work()) + } + } + + /// Blocks until all workers have finished executing their tasks, then returns + /// the set of results. + func waitForResults() -> [TaskResult] { + group.wait() + return resultQueue.sync { results } + } +} diff --git a/Sources/LiteSupport/Run.swift b/Sources/LiteSupport/Run.swift index df9d62d..e54aa9a 100644 --- a/Sources/LiteSupport/Run.swift +++ b/Sources/LiteSupport/Run.swift @@ -22,15 +22,24 @@ import Foundation /// which have valid RUN lines. /// - testLinePrefix: The prefix before `RUN:` in a file. This is almost /// always your specific langauge's line comment syntax. +/// - parallelismLevel: Specifies the amount of parallelism to apply to the +/// test running process. Default value is `.none`, but +/// you can provide `.automatic` to use the available +/// machine cores, or `.explicit(n)` to specify an +/// explicit number of parallel tests. This value should +/// not exceed the number of hyperthreaded cores on your +/// machine, to avoid excessive context switching. /// - Returns: `true` if all tests passed, `false` if any failed. /// - Throws: `LiteError` if there was any issue running tests. public func runLite(substitutions: [(String, String)], pathExtensions: Set, testDirPath: String?, - testLinePrefix: String) throws -> Bool { + testLinePrefix: String, + parallelismLevel: ParallelismLevel = .none) throws -> Bool { let testRunner = try TestRunner(testDirPath: testDirPath, substitutions: substitutions, pathExtensions: pathExtensions, - testLinePrefix: testLinePrefix) + testLinePrefix: testLinePrefix, + parallelismLevel: parallelismLevel) return try testRunner.run() } diff --git a/Sources/LiteSupport/Substitutor.swift b/Sources/LiteSupport/Substitutor.swift index dc027e5..4e6a681 100644 --- a/Sources/LiteSupport/Substitutor.swift +++ b/Sources/LiteSupport/Substitutor.swift @@ -6,6 +6,7 @@ /// available in the repository. import Foundation +import Dispatch public extension String { public var quoted: String { @@ -31,6 +32,7 @@ class Substitutor { let tmpFileRegex = try! NSRegularExpression(pattern: "%t") let tmpDirectoryRegex = try! NSRegularExpression(pattern: "%T") + private let tmpDirQueue = DispatchQueue(label: "tmp-dir-queue") private var tmpDirMap = [URL: URL]() init(substitutions: [(String, String)]) throws { @@ -47,10 +49,12 @@ class Substitutor { } func tempFile(for file: URL) -> URL { - if let tmpFile = tmpDirMap[file] { return tmpFile } - let tmpFile = tempDir.appendingPathComponent(UUID().uuidString) - tmpDirMap[file] = tmpFile - return tmpFile + return tmpDirQueue.sync { + if let tmpFile = tmpDirMap[file] { return tmpFile } + let tmpFile = tempDir.appendingPathComponent(UUID().uuidString) + tmpDirMap[file] = tmpFile + return tmpFile + } } func substitute(_ line: String, in file: URL) -> String { diff --git a/Sources/LiteSupport/TestFile.swift b/Sources/LiteSupport/TestFile.swift index cce74b9..bee3e1b 100644 --- a/Sources/LiteSupport/TestFile.swift +++ b/Sources/LiteSupport/TestFile.swift @@ -6,7 +6,6 @@ /// available in the repository. import Foundation -import SwiftShell /// Represents a test that either passed or failed, and contains the run line /// that triggered the result. @@ -14,8 +13,11 @@ struct TestResult { /// The run line comprising this test. let line: RunLine - /// The output from running this test. - let output: RunOutput + /// The stdout output from running this test. + let stdout: String + + /// The stderr output from running this test. + let stderr: String /// The time it took to execute this test from start to finish. let executionTime: TimeInterval diff --git a/Sources/LiteSupport/TestRunner.swift b/Sources/LiteSupport/TestRunner.swift index 2e1bf6c..eff7cd2 100644 --- a/Sources/LiteSupport/TestRunner.swift +++ b/Sources/LiteSupport/TestRunner.swift @@ -7,7 +7,8 @@ import Foundation import Rainbow -import SwiftShell +import ShellOut +import Dispatch #if os(Linux) /// HACK: This is needed because on macOS, ObjCBool is a distinct type @@ -18,10 +19,31 @@ extension ObjCBool { } #endif +/// Specifies how to parallelize test runs. +public enum ParallelismLevel { + /// Parallelize over an explicit number of cores. + case explicit(Int) + + /// Automatically discover the number of cores on the system and use that. + case automatic + + /// Do not parallelize. + case none + + /// The number of concurrent processes afforded by this level. + var numberOfProcesses: Int { + switch self { + case .explicit(let n): return n + case .none: return 1 + case .automatic: return ProcessInfo.processInfo.processorCount + } + } +} /// TestRunner is responsible for coordinating a set of tests, running them, and /// reporting successes and failures. class TestRunner { + /// The test directory in which tests reside. let testDir: URL @@ -34,11 +56,18 @@ class TestRunner { /// The prefix before `RUN` and `RUN-NOT` lines. let testLinePrefix: String + /// The queue to synchronize printing results. + let resultQueue = DispatchQueue(label: "test-runner-results") + + /// How to parallelize work. + let parallelismLevel: ParallelismLevel + /// Creates a test runner that will execute all tests in the provided /// directory. /// - throws: A LiteError if the test directory is invalid. init(testDirPath: String?, substitutions: [(String, String)], - pathExtensions: Set, testLinePrefix: String) throws { + pathExtensions: Set, testLinePrefix: String, + parallelismLevel: ParallelismLevel) throws { let fm = FileManager.default var isDir: ObjCBool = false let testDirPath = @@ -53,6 +82,7 @@ class TestRunner { self.substitutor = try Substitutor(substitutions: substitutions) self.pathExtensions = pathExtensions self.testLinePrefix = testLinePrefix + self.parallelismLevel = parallelismLevel } func discoverTests() throws -> [TestFile] { @@ -76,32 +106,51 @@ class TestRunner { let files = try discoverTests() if files.isEmpty { return true } - var resultMap = [URL: [TestResult]]() let commonPrefix = files.map { $0.url.path }.commonPrefix - print("Running all tests in \(commonPrefix.bold)") + let workers = parallelismLevel.numberOfProcesses + var testDesc = "Running all tests in \(commonPrefix.bold)" + switch parallelismLevel { + case .automatic, .explicit(_): + testDesc += " across \(workers) threads" + default: break + } + print(testDesc) let prefixLen = commonPrefix.count - var passes = 0 - var failures = 0 - var xFailures = 0 - var time = 0.0 + let executor = ParallelExecutor<[TestResult]>(numberOfWorkers: workers) + let realStart = Date() for file in files { - let results = try run(file: file) - resultMap[file.url] = results - handleResults(file, results: results, prefixLen: prefixLen, - passes: &passes, failures: &failures, - xFailures: &xFailures, time: &time) + executor.addTask { + let results = self.run(file: file) + self.resultQueue.sync { + self.handleResults(file, results: results, prefixLen: prefixLen) + } + return results + } } + let allResults = executor.waitForResults() - printSummary(passes: passes, failures: failures, - xFailures: xFailures, time: time) - return failures == 0 + return printSummary(allResults, realStart: realStart) } - - func printSummary(passes: Int, failures: Int, - xFailures: Int, time: TimeInterval) { - let total = passes + failures + func printSummary(_ results: [[TestResult]], realStart: Date) -> Bool { + var passes = 0, failures = 0, xFailures = 0, total = 0 + var time = 0.0 + let realTime = Date().timeIntervalSince(realStart) + for fileResults in results { + for result in fileResults { + time += result.executionTime + total += 1 + switch result.result { + case .fail: + failures += 1 + case .pass: + passes += 1 + case .xFail: + xFailures += 1 + } + } + } let testDesc = "\(total) test\(total == 1 ? "" : "s")".bold var passDesc = "\(passes) pass\(passes == 1 ? "" : "es")".bold var failDesc = "\(failures) failure\(failures == 1 ? "" : "s")".bold @@ -117,19 +166,25 @@ class TestRunner { xFailDesc = xFailDesc.yellow } let timeStr = time.formatted.cyan.bold - print("Executed \(testDesc) in \(timeStr) with " + - "\(passDesc), \(failDesc), and \(xFailDesc)") + let realTimeStr = realTime.formatted.cyan.bold + print(""" + + \("Total running time:".bold) \(realTimeStr) + \("Total CPU time:".bold) \(timeStr) + Executed \(testDesc) with \(passDesc), \(failDesc), and \(xFailDesc) + + """) if failures == 0 { print("All tests passed! 🎉".green.bold) + return true } + return false } /// Prints individual test results for one specific file. func handleResults(_ file: TestFile, results: [TestResult], - prefixLen: Int, passes: inout Int, - failures: inout Int, xFailures: inout Int, - time: inout TimeInterval) { + prefixLen: Int) { let path = file.url.path let suffixIdx = path.index(path.startIndex, offsetBy: prefixLen, limitedBy: path.endIndex) @@ -145,29 +200,25 @@ class TestRunner { } for result in results { - time += result.executionTime let timeStr = result.executionTime.formatted.cyan switch result.result { case .pass: - passes += 1 print(" \("✔".green.bold) \(result.line.asString) (\(timeStr))") case .xFail: - xFailures += 1 print(" ⚠️ \(result.line.asString) (\(timeStr))") case .fail: - failures += 1 print(" \("𝗫".red.bold) \(result.line.asString) (\(timeStr))") print(" exit status: \(result.exitStatus)") - if !result.output.stderror.isEmpty { + if !result.stderr.isEmpty { print(" stderr:") - let lines = result.output.stderror.split(separator: "\n") - .joined(separator: "\n ") + let lines = result.stderr.split(separator: "\n") + .joined(separator: "\n ") print(" \(lines)") } - if !result.output.stdout.isEmpty { + if !result.stdout.isEmpty { print(" stdout:") - let lines = result.output.stdout.split(separator: "\n") - .joined(separator: "\n ") + let lines = result.stdout.split(separator: "\n") + .joined(separator: "\n ") print(" \(lines)") } print(" command line:") @@ -180,18 +231,32 @@ class TestRunner { /// Runs all the run lines in a given file and returns a test result /// with the individual successes or failures. - private func run(file: TestFile) throws -> [TestResult] { + private func run(file: TestFile) -> [TestResult] { var results = [TestResult]() for line in file.runLines { let start = Date() + let stdout: String + let stderr: String + let exitCode: Int let bash = file.makeCommandLine(line, substitutor: substitutor) - let output = SwiftShell.main.run(bash: bash) + do { + stdout = try shellOut(to: bash) + stderr = "" + exitCode = 0 + } catch let error as ShellOutError { + stderr = error.message + stdout = error.output + exitCode = Int(error.terminationStatus) + } catch { + fatalError("unhandled error") + } let end = Date() results.append(TestResult(line: line, - output: output, + stdout: stdout, + stderr: stderr, executionTime: end.timeIntervalSince(start), file: file.url, - exitStatus: Int(output.exitcode))) + exitStatus: exitCode)) } return results } diff --git a/Sources/lite-test/main.swift b/Sources/lite-test/main.swift index 3809577..bfd464c 100644 --- a/Sources/lite-test/main.swift +++ b/Sources/lite-test/main.swift @@ -9,7 +9,8 @@ do { let allPassed = try runLite(substitutions: [("echo", "echo")], pathExtensions: ["test"], testDirPath: nil, - testLinePrefix: "//") + testLinePrefix: "//", + parallelismLevel: .automatic) exit(allPassed ? 0 : -1) } catch let err as LiteError { fputs("error: \(err.message)", stderr) diff --git a/Tests/LinuxMain.swift b/Tests/LinuxMain.swift index 0519f29..f524ae9 100644 --- a/Tests/LinuxMain.swift +++ b/Tests/LinuxMain.swift @@ -2,5 +2,5 @@ import XCTest @testable import LiteTests XCTMain([ - testCase(LiteTests.allTests), + LiteTests.allTests, ])