diff --git a/benchmark/utils/ArgParse.swift b/benchmark/utils/ArgParse.swift index 96b290d015787..8796e02d6871b 100644 --- a/benchmark/utils/ArgParse.swift +++ b/benchmark/utils/ArgParse.swift @@ -12,67 +12,227 @@ import Foundation -public struct Arguments { - public var progName: String - public var positionalArgs: [String] - public var optionalArgsMap: [String : String] - - init(_ pName: String, _ posArgs: [String], _ optArgsMap: [String : String]) { - progName = pName - positionalArgs = posArgs - optionalArgsMap = optArgsMap +enum ArgumentError: Error { + case missingValue(String) + case invalidType(value: String, type: String, argument: String?) + case unsupportedArgument(String) +} + +extension ArgumentError: CustomStringConvertible { + public var description: String { + switch self { + case let .missingValue(key): + return "missing value for '\(key)'" + case let .invalidType(value, type, argument): + return (argument == nil) + ? "'\(value)' is not a valid '\(type)'" + : "'\(value)' is not a valid '\(type)' for '\(argument!)'" + case let .unsupportedArgument(argument): + return "unsupported argument '\(argument)'" + } } } -/// Using CommandLine.arguments, returns an Arguments struct describing -/// the arguments to this program. If we fail to parse arguments, we -/// return nil. +/// Type-checked parsing of the argument value. /// -/// We assume that optional switch args are of the form: +/// - Returns: Typed value of the argument converted using the `parse` function. /// -/// --opt-name[=opt-value] -/// -opt-name[=opt-value] -/// -/// with opt-name and opt-value not containing any '=' signs. Any -/// other option passed in is assumed to be a positional argument. -public func parseArgs(_ validOptions: [String]? = nil) - -> Arguments? { - let progName = CommandLine.arguments[0] - var positionalArgs = [String]() - var optionalArgsMap = [String : String]() - - // For each argument we are passed... - var passThroughArgs = false - for arg in CommandLine.arguments[1..( + _ parse: (String) throws -> T?, + _ value: String, + argument: String? = nil +) throws -> T { + if let t = try parse(value) { return t } + var type = "\(T.self)" + if type.starts(with: "Optional<") { + let s = type.index(after: type.index(of:"<")!) + let e = type.index(before: type.endIndex) // ">" + type = String(type[s ..< e]) // strip Optional< > + } + throw ArgumentError.invalidType( + value: value, type: type, argument: argument) +} + +/// Parser that converts the program's command line arguments to typed values +/// according to the parser's configuration, storing them in the provided +/// instance of a value-holding type. +class ArgumentParser { + private var result: U + private var validOptions: [String] { + return arguments.compactMap { $0.name } + } + private var arguments: [Argument] = [] + private let programName: String = { + // Strip full path from the program name. + let r = CommandLine.arguments[0].reversed() + let ss = r[r.startIndex ..< (r.index(of:"/") ?? r.endIndex)] + return String(ss.reversed()) + }() + private var positionalArgs = [String]() + private var optionalArgsMap = [String : String]() + + /// Argument holds the name of the command line parameter, its help + /// desciption and a rule that's applied to process it. + /// + /// The the rule is typically a value processing closure used to convert it + /// into given type and storing it in the parsing result. + /// + /// See also: addArgument, parseArgument + struct Argument { + let name: String? + let help: String? + let apply: () throws -> () + } + + /// ArgumentParser is initialized with an instance of a type that holds + /// the results of the parsing of the individual command line arguments. + init(into result: U) { + self.result = result + self.arguments += [ + Argument(name: "--help", help: "show this help message and exit", + apply: printUsage) + ] } - if arg == "--" { - passThroughArgs = true - continue + + private func printUsage() { + guard let _ = optionalArgsMap["--help"] else { return } + let space = " " + let maxLength = arguments.compactMap({ $0.name?.count }).max()! + let padded = { (s: String) in + " \(s)\(String(repeating:space, count: maxLength - s.count)) " } + let f: (String, String) -> String = { + "\(padded($0))\($1)" + .split(separator: "\n") + .joined(separator: "\n" + padded("")) + } + let positional = f("TEST", "name or number of the benchmark to measure") + let optional = arguments.filter { $0.name != nil } + .map { f($0.name!, $0.help ?? "") } + .joined(separator: "\n") + print( + """ + usage: \(programName) [--argument=VALUE] [TEST [TEST ...]] + + positional arguments: + \(positional) + + optional arguments: + \(optional) + """) + exit(0) } - // Attempt to split it into two components separated by an equals sign. - let components = arg.split(separator: "=") - let optionName = String(components[0]) - if validOptions != nil && !validOptions!.contains(optionName) { - print("Invalid option: \(arg)") - return nil + + /// Parses the command line arguments, returning the result filled with + /// specified argument values or report errors and exit the program if + /// the parsing fails. + public func parse() -> U { + do { + try parseArgs() // parse the argument syntax + try arguments.forEach { try $0.apply() } // type-check and store values + return result + } catch let error as ArgumentError { + fputs("error: \(error)\n", stderr) + exit(1) + } catch { + fflush(stdout) + fatalError("\(error)") + } } - var optionVal : String - switch components.count { - case 1: optionVal = "" - case 2: optionVal = String(components[1]) - default: - // If we do not have two components at this point, we can not have - // an option switch. This is an invalid argument. Bail! - print("Invalid option: \(arg)") - return nil + + /// Using CommandLine.arguments, parses the structure of optional and + /// positional arguments of this program. + /// + /// We assume that optional switch args are of the form: + /// + /// --opt-name[=opt-value] + /// -opt-name[=opt-value] + /// + /// with `opt-name` and `opt-value` not containing any '=' signs. Any + /// other option passed in is assumed to be a positional argument. + /// + /// - Throws: `ArgumentError.unsupportedArgument` on failure to parse + /// the supported argument syntax. + private func parseArgs() throws { + + // For each argument we are passed... + for arg in CommandLine.arguments[1..( + _ name: String?, + _ property: WritableKeyPath, + defaultValue: T? = nil, + help: String? = nil, + parser: @escaping (String) throws -> T? = { _ in nil } + ) { + arguments.append(Argument(name: name, help: help) + { try self.parseArgument(name, property, defaultValue, parser) }) } - optionalArgsMap[optionName] = optionVal - } - return Arguments(progName, positionalArgs, optionalArgsMap) + /// Process the specified command line argument. + /// + /// For optional arguments that have a value we attempt to convert it into + /// given type using the supplied parser, performing the type-checking with + /// the `checked` function. + /// If the value is empty the `defaultValue` is used instead. + /// The typed value is finally stored in the `result` into the specified + /// `property`. + /// + /// For the optional positional arguments, the [String] is simply assigned + /// to the specified property without any conversion. + /// + /// See `addArgument` for detailed parameter descriptions. + private func parseArgument( + _ name: String?, + _ property: WritableKeyPath, + _ defaultValue: T?, + _ parse: (String) throws -> T? + ) throws { + if let name = name, let value = optionalArgsMap[name] { + guard !value.isEmpty || defaultValue != nil + else { throw ArgumentError.missingValue(name) } + + result[keyPath: property] = (value.isEmpty) + ? defaultValue! + : try checked(parse, value, argument: name) + } else if name == nil { + result[keyPath: property] = positionalArgs as! T + } + } } diff --git a/benchmark/utils/DriverUtils.swift b/benchmark/utils/DriverUtils.swift index 5675c767ecd9b..3465b855ea01f 100644 --- a/benchmark/utils/DriverUtils.swift +++ b/benchmark/utils/DriverUtils.swift @@ -19,230 +19,176 @@ import Darwin import TestsUtils struct BenchResults { - var delim: String = "," - var sampleCount: UInt64 = 0 - var min: UInt64 = 0 - var max: UInt64 = 0 - var mean: UInt64 = 0 - var sd: UInt64 = 0 - var median: UInt64 = 0 - - init() {} - - init(delim: String, sampleCount: UInt64, min: UInt64, max: UInt64, mean: UInt64, sd: UInt64, median: UInt64) { - self.delim = delim - self.sampleCount = sampleCount - self.min = min - self.max = max - self.mean = mean - self.sd = sd - self.median = median - - // Sanity the bounds of our results - precondition(self.min <= self.max, "min should always be <= max") - precondition(self.min <= self.mean, "min should always be <= mean") - precondition(self.min <= self.median, "min should always be <= median") - precondition(self.max >= self.mean, "max should always be >= mean") - precondition(self.max >= self.median, "max should always be >= median") - } + let sampleCount, min, max, mean, sd, median, maxRSS: UInt64 } -extension BenchResults : CustomStringConvertible { - var description: String { - return "\(sampleCount)\(delim)\(min)\(delim)\(max)\(delim)\(mean)\(delim)\(sd)\(delim)\(median)" - } -} - -struct Test { - let benchInfo: BenchmarkInfo - let index: Int - - /// The name of the benchmark. - var name: String { - return benchInfo.name - } - - /// The "main routine" of the benchmark. - var runFunction: ((Int) -> ())? { - return benchInfo.runFunction - } - - /// The benchmark categories that this test belongs to. Used for filtering. - var tags: [BenchmarkCategory] { - return benchInfo.tags.sorted() - } - - /// An optional initialization function for a benchmark that is run before - /// measuring begins. Intended to be used to initialize global data used in - /// a benchmark. - var setUpFunction: (() -> ())? { - return benchInfo.setUpFunction - } - - /// An optional deinitialization function that if non-null is run /after/ a - /// measurement has been taken. - var tearDownFunction: (() -> ())? { - return benchInfo.tearDownFunction - } -} - -// We should migrate to a collection of BenchmarkInfo. public var registeredBenchmarks: [BenchmarkInfo] = [] enum TestAction { case run case listTests - case fail(String) - case help([String]) } struct TestConfig { /// The delimiter to use when printing output. - var delim: String = "," - - /// The filters applied to our test names. - var filters = [String]() - - /// The tags that we want to run - var tags = Set() - - /// Tests tagged with any of these will not be executed - var skipTags: Set = [.unstable, .skip] + let delim: String - /// The scalar multiple of the amount of times a test should be run. This - /// enables one to cause tests to run for N iterations longer than they - /// normally would. This is useful when one wishes for a test to run for a + /// Duration of the test measurement in seconds. + /// + /// Used to compute the number of iterations, if no fixed amount is specified. + /// This is useful when one wishes for a test to run for a /// longer amount of time to perform performance analysis on the test in /// instruments. - var iterationScale: Int = 1 + let sampleTime: Double /// If we are asked to have a fixed number of iterations, the number of fixed - /// iterations. - var fixedNumIters: UInt = 0 + /// iterations. The default value of 0 means: automatically compute the + /// number of iterations to measure the test for a specified sample time. + let fixedNumIters: UInt /// The number of samples we should take of each test. - var numSamples: Int = 1 + let numSamples: Int /// Is verbose output enabled? - var verbose: Bool = false + let verbose: Bool + + // Should we log the test's memory usage? + let logMemory: Bool /// After we run the tests, should the harness sleep to allow for utilities /// like leaks that require a PID to run on the test harness. - var afterRunSleep: Int? + let afterRunSleep: Int? /// The list of tests to run. - var tests = [Test]() - - mutating func processArguments() -> TestAction { - let validOptions = [ - "--iter-scale", "--num-samples", "--num-iters", - "--verbose", "--delim", "--list", "--sleep", - "--tags", "--skip-tags", "--help" - ] - let maybeBenchArgs: Arguments? = parseArgs(validOptions) - if maybeBenchArgs == nil { - return .fail("Failed to parse arguments") - } - let benchArgs = maybeBenchArgs! - - filters = benchArgs.positionalArgs - - if benchArgs.optionalArgsMap["--help"] != nil { - return .help(validOptions) - } - - if let x = benchArgs.optionalArgsMap["--iter-scale"] { - if x.isEmpty { return .fail("--iter-scale requires a value") } - iterationScale = Int(x)! - } - - if let x = benchArgs.optionalArgsMap["--num-iters"] { - if x.isEmpty { return .fail("--num-iters requires a value") } - fixedNumIters = numericCast(Int(x)!) - } - - if let x = benchArgs.optionalArgsMap["--num-samples"] { - if x.isEmpty { return .fail("--num-samples requires a value") } - numSamples = Int(x)! + let tests: [(index: String, info: BenchmarkInfo)] + + let action: TestAction + + init(_ registeredBenchmarks: [BenchmarkInfo]) { + + struct PartialTestConfig { + var delim: String? + var tags, skipTags: Set? + var numSamples, afterRunSleep: Int? + var fixedNumIters: UInt? + var sampleTime: Double? + var verbose: Bool? + var logMemory: Bool? + var action: TestAction? + var tests: [String]? } - if let _ = benchArgs.optionalArgsMap["--verbose"] { - verbose = true - print("Verbose") - } - - if let x = benchArgs.optionalArgsMap["--delim"] { - if x.isEmpty { return .fail("--delim requires a value") } - delim = x - } - - if let x = benchArgs.optionalArgsMap["--tags"] { - if x.isEmpty { return .fail("--tags requires a value") } - + // Custom value type parsers + func tags(tags: String) throws -> Set { // We support specifying multiple tags by splitting on comma, i.e.: - // // --tags=Array,Dictionary - // - // FIXME: If we used Error instead of .fail, then we could have a cleaner - // impl here using map on x and tags.formUnion. - for t in x.split(separator: ",") { - guard let cat = BenchmarkCategory(rawValue: String(t)) else { - return .fail("Unknown benchmark category: '\(t)'") - } - tags.insert(cat) - } - } - - if let x = benchArgs.optionalArgsMap["--skip-tags"] { - // if the --skip-tags parameter is specified, we need to ignore the - // default and start from a clean slate. - skipTags = [] - - // We support specifying multiple tags by splitting on comma, i.e.: - // // --skip-tags=Array,Set,unstable,skip - // - // FIXME: If we used Error instead of .fail, then we could have a cleaner - // impl here using map on x and tags.formUnion. - for t in x.split(separator: ",") { - guard let cat = BenchmarkCategory(rawValue: String(t)) else { - return .fail("Unknown benchmark category: '\(t)'") - } - skipTags.insert(cat) - } + return Set( + try tags.split(separator: ",").map(String.init).map { + try checked({ BenchmarkCategory(rawValue: $0) }, $0) }) } - - if let x = benchArgs.optionalArgsMap["--sleep"] { - guard let v = Int(x) else { - return .fail("--sleep requires a non-empty integer value") - } - afterRunSleep = v + func finiteDouble(value: String) -> Double? { + return Double(value).flatMap { $0.isFinite ? $0 : nil } } - if let _ = benchArgs.optionalArgsMap["--list"] { - return .listTests + // Configure the command line argument parser + let p = ArgumentParser(into: PartialTestConfig()) + p.addArgument("--num-samples", \.numSamples, + help: "number of samples to take per benchmark; default: 1", + parser: { Int($0) }) + p.addArgument("--num-iters", \.fixedNumIters, + help: "number of iterations averaged in the sample;\n" + + "default: auto-scaled to measure for `sample-time`", + parser: { UInt($0) }) + p.addArgument("--sample-time", \.sampleTime, + help: "duration of test measurement in seconds\ndefault: 1", + parser: finiteDouble) + p.addArgument("--verbose", \.verbose, defaultValue: true, + help: "increase output verbosity") + p.addArgument("--memory", \.logMemory, defaultValue: true, + help: "log the change in maximum resident set size (MAX_RSS)") + p.addArgument("--delim", \.delim, + help:"value delimiter used for log output; default: ,", + parser: { $0 }) + p.addArgument("--tags", \PartialTestConfig.tags, + help: "run tests matching all the specified categories", + parser: tags) + p.addArgument("--skip-tags", \PartialTestConfig.skipTags, defaultValue: [], + help: "don't run tests matching any of the specified\n" + + "categories; default: unstable,skip", + parser: tags) + p.addArgument("--sleep", \.afterRunSleep, + help: "number of seconds to sleep after benchmarking", + parser: { Int($0) }) + p.addArgument("--list", \.action, defaultValue: .listTests, + help: "don't run the tests, just log the list of test \n" + + "numbers, names and tags (respects specified filters)") + p.addArgument(nil, \.tests) // positional arguments + + let c = p.parse() + + // Configure from the command line arguments, filling in the defaults. + delim = c.delim ?? "," + sampleTime = c.sampleTime ?? 1.0 + fixedNumIters = c.fixedNumIters ?? 0 + numSamples = c.numSamples ?? 1 + verbose = c.verbose ?? false + logMemory = c.logMemory ?? false + afterRunSleep = c.afterRunSleep + action = c.action ?? .run + tests = TestConfig.filterTests(registeredBenchmarks, + specifiedTests: Set(c.tests ?? []), + tags: c.tags ?? [], + skipTags: c.skipTags ?? [.unstable, .skip]) + + if verbose { + let testList = tests.map({ $0.1.name }).joined(separator: ", ") + print(""" + --- CONFIG --- + NumSamples: \(numSamples) + Verbose: \(verbose) + LogMemory: \(logMemory) + SampleTime: \(sampleTime) + FixedIters: \(fixedNumIters) + Tests Filter: \(c.tests ?? []) + Tests to run: \(testList) + + --- DATA ---\n + """) } - - return .run } - mutating func findTestsToRun() { - registeredBenchmarks.sort() + /// Returns the list of tests to run. + /// + /// - Parameters: + /// - registeredBenchmarks: List of all performance tests to be filtered. + /// - specifiedTests: List of explicitly specified tests to run. These can be + /// specified either by a test name or a test number. + /// - tags: Run tests tagged with all of these categories. + /// - skipTags: Don't run tests tagged with any of these categories. + /// - Returns: An array of test number and benchmark info tuples satisfying + /// specified filtering conditions. + static func filterTests( + _ registeredBenchmarks: [BenchmarkInfo], + specifiedTests: Set, + tags: Set, + skipTags: Set + ) -> [(index: String, info: BenchmarkInfo)] { let indices = Dictionary(uniqueKeysWithValues: - zip(registeredBenchmarks.map{$0.name}, 1...)) - let benchmarkNamesOrIndices = Set(filters) - // needed so we don't capture an ivar of a mutable inout self. - let (_tags, _skipTags) = (tags, skipTags) - - tests = registeredBenchmarks.filter { benchmark in - if benchmarkNamesOrIndices.isEmpty { - return benchmark.tags.isSuperset(of: _tags) && - benchmark.tags.isDisjoint(with: _skipTags) - } else { - return benchmarkNamesOrIndices.contains(benchmark.name) || - benchmarkNamesOrIndices.contains(String(indices[benchmark.name]!)) - } - }.map { Test(benchInfo: $0, index: indices[$0.name]!) } + zip(registeredBenchmarks.sorted().map { $0.name }, + (1...).lazy.map { String($0) } )) + + func byTags(b: BenchmarkInfo) -> Bool { + return b.tags.isSuperset(of: tags) && + b.tags.isDisjoint(with: skipTags) + } + func byNamesOrIndices(b: BenchmarkInfo) -> Bool { + return specifiedTests.contains(b.name) || + specifiedTests.contains(indices[b.name]!) + } // !! "All registeredBenchmarks have been assigned an index" + return registeredBenchmarks + .filter(specifiedTests.isEmpty ? byTags : byNamesOrIndices) + .map { (index: indices[$0.name]!, info: $0) } } } @@ -327,6 +273,37 @@ class Timer { class SampleRunner { let timer = Timer() + let baseline = SampleRunner.getResourceUtilization() + let c: TestConfig + + init(_ config: TestConfig) { + self.c = config + } + + private static func getResourceUtilization() -> rusage { + var u = rusage(); getrusage(RUSAGE_SELF, &u); return u + } + + /// Returns maximum resident set size (MAX_RSS) delta in bytes + func measureMemoryUsage() -> Int { + var current = SampleRunner.getResourceUtilization() + let maxRSS = current.ru_maxrss - baseline.ru_maxrss + + if c.verbose { + let pages = maxRSS / sysconf(_SC_PAGESIZE) + func deltaEquation(_ stat: KeyPath) -> String { + let b = baseline[keyPath: stat], c = current[keyPath: stat] + return "\(c) - \(b) = \(c - b)" + } + print(""" + MAX_RSS \(deltaEquation(\rusage.ru_maxrss)) (\(pages) pages) + ICS \(deltaEquation(\rusage.ru_nivcsw)) + VCS \(deltaEquation(\rusage.ru_nvcsw)) + """) + } + return maxRSS + } + func run(_ name: String, fn: (Int) -> Void, num_iters: UInt) -> UInt64 { // Start the timer. #if SWIFT_RUNTIME_ENABLE_LEAK_CHECKER @@ -346,7 +323,7 @@ class SampleRunner { } /// Invoke the benchmark entry point and return the run time in milliseconds. -func runBench(_ test: Test, _ c: TestConfig) -> BenchResults? { +func runBench(_ test: BenchmarkInfo, _ c: TestConfig) -> BenchResults? { var samples = [UInt64](repeating: 0, count: c.numSamples) // Before we do anything, check that we actually have a function to @@ -363,10 +340,11 @@ func runBench(_ test: Test, _ c: TestConfig) -> BenchResults? { print("Running \(test.name) for \(c.numSamples) samples.") } - let sampler = SampleRunner() + let sampler = SampleRunner(c) for s in 0.. BenchResults? { let (mean, sd) = internalMeanSD(samples) // Return our benchmark results. - return BenchResults(delim: c.delim, sampleCount: UInt64(samples.count), + return BenchResults(sampleCount: UInt64(samples.count), min: samples.min()!, max: samples.max()!, - mean: mean, sd: sd, median: internalMedian(samples)) -} - -func printRunInfo(_ c: TestConfig) { - if c.verbose { - print("--- CONFIG ---") - print("NumSamples: \(c.numSamples)") - print("Verbose: \(c.verbose)") - print("IterScale: \(c.iterationScale)") - if c.fixedNumIters != 0 { - print("FixedIters: \(c.fixedNumIters)") - } - print("Tests Filter: \(c.filters)") - print("Tests to run: ", terminator: "") - for t in c.tests { - print("\(t.name), ", terminator: "") - } - print("") - print("") - print("--- DATA ---") - } + mean: mean, sd: sd, median: internalMedian(samples), + maxRSS: UInt64(sampler.measureMemoryUsage())) } +/// Execute benchmarks and continuously report the measurement results. func runBenchmarks(_ c: TestConfig) { - let units = "us" - print("#\(c.delim)TEST\(c.delim)SAMPLES\(c.delim)MIN(\(units))\(c.delim)MAX(\(units))\(c.delim)MEAN(\(units))\(c.delim)SD(\(units))\(c.delim)MEDIAN(\(units))") - var sumBenchResults = BenchResults() - sumBenchResults.sampleCount = 0 - - for t in c.tests { - guard let results = runBench(t, c) else { - print("\(t.index)\(c.delim)\(t.name)\(c.delim)Unsupported") - fflush(stdout) - continue + let withUnit = {$0 + "(us)"} + let header = ( + ["#", "TEST", "SAMPLES"] + + ["MIN", "MAX", "MEAN", "SD", "MEDIAN"].map(withUnit) + + (c.logMemory ? ["MAX_RSS(B)"] : []) + ).joined(separator: c.delim) + print(header) + + var testCount = 0 + + func report(_ index: String, _ t: BenchmarkInfo, results: BenchResults?) { + func values(r: BenchResults) -> [String] { + return ([r.sampleCount, r.min, r.max, r.mean, r.sd, r.median] + + (c.logMemory ? [r.maxRSS] : [])).map { String($0) } } - print("\(t.index)\(c.delim)\(t.name)\(c.delim)\(results.description)") + let benchmarkStats = ( + [index, t.name] + (results.map(values) ?? ["Unsupported"]) + ).joined(separator: c.delim) + + print(benchmarkStats) fflush(stdout) - sumBenchResults.min += results.min - sumBenchResults.max += results.max - sumBenchResults.mean += results.mean - sumBenchResults.sampleCount += 1 - // Don't accumulate SD and Median, as simple sum isn't valid for them. - // TODO: Compute SD and Median for total results as well. - // sumBenchResults.sd += results.sd - // sumBenchResults.median += results.median + if (results != nil) { + testCount += 1 + } + } + + for (index, test) in c.tests { + report(index, test, results:runBench(test, c)) } print("") - print("Totals\(c.delim)\(sumBenchResults.description)") + print("Totals\(c.delim)\(testCount)") } public func main() { - var config = TestConfig() - - switch (config.processArguments()) { - case let .help(validOptions): - print("Valid options:") - for v in validOptions { - print(" \(v)") - } - case let .fail(msg): - // We do this since we need an autoclosure... - fatalError("\(msg)") - case .listTests: - config.findTestsToRun() - print("#\(config.delim)Test\(config.delim)[Tags]") - for t in config.tests { - print("\(t.index)\(config.delim)\(t.name)\(config.delim)\(t.tags)") - } - case .run: - config.findTestsToRun() - printRunInfo(config) - runBenchmarks(config) - if let x = config.afterRunSleep { - sleep(UInt32(x)) - } + let config = TestConfig(registeredBenchmarks) + switch (config.action) { + case .listTests: + print("#\(config.delim)Test\(config.delim)[Tags]") + for (index, t) in config.tests { + let testDescription = [String(index), t.name, t.tags.sorted().description] + .joined(separator: config.delim) + print(testDescription) + } + case .run: + runBenchmarks(config) + if let x = config.afterRunSleep { + sleep(UInt32(x)) + } } } diff --git a/test/benchmark/Benchmark_O.test-sh b/test/benchmark/Benchmark_O.test-sh deleted file mode 100644 index 7e991e9240c2d..0000000000000 --- a/test/benchmark/Benchmark_O.test-sh +++ /dev/null @@ -1,50 +0,0 @@ -// REQUIRES: OS=macosx -// REQUIRES: asserts -// REQUIRES: benchmark -// REQUIRES: CMAKE_GENERATOR=Ninja - -// RUN: %Benchmark_O --list | %FileCheck %s --check-prefix LISTTAGS -// LISTTAGS: AngryPhonebook,[ -// LISTTAGS-NOT: TestsUtils.BenchmarkCategory. -// LISTTAGS-SAME: String, -// LISTTAGS-SAME: ] - -// RUN: %Benchmark_O AngryPhonebook --num-iters=1 \ -// RUN: | %FileCheck %s --check-prefix NUMITERS1 -// NUMITERS1: AngryPhonebook,1 -// NUMITERS1-NOT: 0,0,0,0,0 - -// Should run benchmark by name, even if its tags match the default skip-tags -// (unstable,skip). Ackermann is marked unstable -// RUN: %Benchmark_O Ackermann | %FileCheck %s --check-prefix NAMEDSKIP -// NAMEDSKIP: Ackermann - -// RUN: %Benchmark_O --list --tags=Dictionary,Array \ -// RUN: | %FileCheck %s --check-prefix ANDTAGS -// ANDTAGS: TwoSum -// ANDTAGS-NOT: Array2D -// ANDTAGS-NOT: DictionarySwap - -// RUN: %Benchmark_O --list --tags=algorithm --skip-tags=validation \ -// RUN: | %FileCheck %s --check-prefix TAGSANDSKIPTAGS -// TAGSANDSKIPTAGS: Ackermann -// TAGSANDSKIPTAGS: DictOfArraysToArrayOfDicts -// TAGSANDSKIPTAGS: Fibonacci -// TAGSANDSKIPTAGS: RomanNumbers - -// RUN: %Benchmark_O --list --tags=algorithm \ -// RUN: --skip-tags=validation,Dictionary,String \ -// RUN: | %FileCheck %s --check-prefix ORSKIPTAGS -// ORSKIPTAGS: Ackermann -// ORSKIPTAGS-NOT: DictOfArraysToArrayOfDicts -// ORSKIPTAGS: Fibonacci -// ORSKIPTAGS-NOT: RomanNumbers - -// RUN: %Benchmark_O --list | %FileCheck %s --check-prefix LISTPRECOMMIT -// LISTPRECOMMIT: #,Test,[Tags] -// LISTPRECOMMIT-NOT: Ackermann -// LISTPRECOMMIT: {{[0-9]+}},AngryPhonebook - -// RUN: %Benchmark_O --list --skip-tags= | %FileCheck %s --check-prefix LISTALL -// LISTALL: Ackermann -// LISTALL: AngryPhonebook diff --git a/test/benchmark/Benchmark_O.test.md b/test/benchmark/Benchmark_O.test.md new file mode 100644 index 0000000000000..865299afd8e4c --- /dev/null +++ b/test/benchmark/Benchmark_O.test.md @@ -0,0 +1,190 @@ + +# `Benchmark_O` Tests + +The `Benchmark_O` binary is used directly from command line as well as a +subcomponent invoked from higher-level scripts (eg. [`Benchmark_Driver`][BD]). +These script therefore depend on the supported command line options and the +format of its console output. The following [`lit` tests][Testing] also serve +as a verification of this public API to prevent its accidental breakage. + +[BD]: https://github.com/apple/swift/blob/master/benchmark/scripts/Benchmark_Driver +[Testing]: https://github.com/apple/swift/blob/master/docs/Testing.md + +Note: Following tests use *Ackermann* as an example of a benchmark that is +excluded from the default "pre-commit" list because it is marked `unstable` and +the default skip-tags (`unstable,skip`) will exclude it. It's also +alphabetically the first benchmark in the test suite (used to verify running by +index). If these assumptions change, the test must be adapted. + +## List Format +```` +RUN: %Benchmark_O --list | %FileCheck %s \ +RUN: --check-prefix LISTPRECOMMIT \ +RUN: --check-prefix LISTTAGS +LISTPRECOMMIT: #,Test,[Tags] +LISTPRECOMMIT-NOT: Ackermann +LISTPRECOMMIT: {{[0-9]+}},AngryPhonebook +LISTTAGS-SAME: ,[ +LISTTAGS-NOT: TestsUtils.BenchmarkCategory. +LISTTAGS-SAME: String, api, validation +LISTTAGS-SAME: ] +```` + +Verify Ackermann is listed when skip-tags are explicitly empty and that it is +marked unstable: + +```` +RUN: %Benchmark_O --list --skip-tags= | %FileCheck %s --check-prefix LISTALL +LISTALL: Ackermann +LISTALL-SAME: unstable +LISTALL: AngryPhonebook +```` + +## Benchmark Selection +The logic for filtering tests based on specified names, indices and tags +is shared between the default "run" and `--list` commands. It is tested on +the list command, which is much faster, because it runs no benchmarks. +It provides us with ability to do a "dry run". + +Run benchmark by name (even if its tags match the skip-tags) or test number: + +```` +RUN: %Benchmark_O Ackermann --list | %FileCheck %s --check-prefix NAMEDSKIP +NAMEDSKIP: Ackermann + +RUN: %Benchmark_O 1 --list | %FileCheck %s --check-prefix RUNBYNUMBER +RUNBYNUMBER: Ackermann +```` + +Composition of `tags` and `skip-tags`: + +```` +RUN: %Benchmark_O --list --tags=Dictionary,Array \ +RUN: | %FileCheck %s --check-prefix ANDTAGS +ANDTAGS: TwoSum +ANDTAGS-NOT: Array2D +ANDTAGS-NOT: DictionarySwap + +RUN: %Benchmark_O --list --tags=algorithm --skip-tags=validation \ +RUN: | %FileCheck %s --check-prefix TAGSANDSKIPTAGS +TAGSANDSKIPTAGS: Ackermann +TAGSANDSKIPTAGS: DictOfArraysToArrayOfDicts +TAGSANDSKIPTAGS: Fibonacci +TAGSANDSKIPTAGS: RomanNumbers + +RUN: %Benchmark_O --list --tags=algorithm \ +RUN: --skip-tags=validation,Dictionary,String \ +RUN: | %FileCheck %s --check-prefix ORSKIPTAGS +ORSKIPTAGS: Ackermann +ORSKIPTAGS-NOT: DictOfArraysToArrayOfDicts +ORSKIPTAGS: Fibonacci +ORSKIPTAGS-NOT: RomanNumbers +```` + +## Running Benchmarks +Each real benchmark execution takes about a second per sample. If possible, +multiple checks are combined into one run to minimize the test time. + +```` +RUN: %Benchmark_O AngryPhonebook --num-iters=1 \ +RUN: | %FileCheck %s --check-prefix NUMITERS1 \ +RUN: --check-prefix LOGHEADER \ +RUN: --check-prefix LOGBENCH +LOGHEADER-LABEL: #,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us) +LOGBENCH: {{[0-9]+}}, +NUMITERS1: AngryPhonebook,1 +NUMITERS1-NOT: 0,0,0,0,0 +LOGBENCH-SAME: ,{{[0-9]+}},{{[0-9]+}},{{[0-9]+}},{{[0-9]+}},{{[0-9]+}} +```` + +### Verbose Mode + +```` +RUN: %Benchmark_O 1 Ackermann 1 AngryPhonebook \ +RUN: --verbose --num-samples=2 --memory \ +RUN: | %FileCheck %s --check-prefix RUNJUSTONCE \ +RUN: --check-prefix CONFIG \ +RUN: --check-prefix LOGVERBOSE \ +RUN: --check-prefix MEASUREENV \ +RUN: --check-prefix LOGMEMORY +CONFIG: NumSamples: 2 +CONFIG: Tests Filter: ["1", "Ackermann", "1", "AngryPhonebook"] +CONFIG: Tests to run: Ackermann, AngryPhonebook +LOGMEMORY: #,TEST,SAMPLES,MIN(us),MAX(us),MEAN(us),SD(us),MEDIAN(us),MAX_RSS(B) +LOGVERBOSE-LABEL: Running Ackermann for 2 samples. +LOGVERBOSE: Measuring with scale {{[0-9]+}}. +LOGVERBOSE-NEXT: Sample 0,{{[0-9]+}} +LOGVERBOSE-NEXT: Measuring with scale {{[0-9]+}}. +LOGVERBOSE-NEXT: Sample 1,{{[0-9]+}} +MEASUREENV: MAX_RSS {{[0-9]+}} - {{[0-9]+}} = {{[0-9]+}} ({{[0-9]+}} pages) +MEASUREENV: ICS {{[0-9]+}} - {{[0-9]+}} = {{[0-9]+}} +MEASUREENV: VCS {{[0-9]+}} - {{[0-9]+}} = {{[0-9]+}} +RUNJUSTONCE-LABEL: 1,Ackermann +RUNJUSTONCE-NOT: 1,Ackermann +LOGMEMORY: ,{{[0-9]+}},{{[0-9]+}},{{[0-9]+}},{{[0-9]+}},{{[0-9]+}},{{[0-9]+}} +LOGVERBOSE-LABEL: Running AngryPhonebook for 2 samples. +```` + +## Error Handling + +```` +RUN: not %Benchmark_O --bogus 2>&1 \ +RUN: | %FileCheck %s --check-prefix ARGPARSE +ARGPARSE: error: unsupported argument '--bogus' + +RUN: not %Benchmark_O --sample-time \ +RUN: 2>&1 | %FileCheck %s --check-prefix NOVALUE +NOVALUE: error: missing value for '--sample-time' + +RUN: not %Benchmark_O --sample-time= \ +RUN: 2>&1 | %FileCheck %s --check-prefix EMPTYVAL +EMPTYVAL: error: missing value for '--sample-time' + +RUN: not %Benchmark_O --sample-time=NaN \ +RUN: 2>&1 | %FileCheck %s --check-prefix NANVALUE +NANVALUE: error: 'NaN' is not a valid 'Double' for '--sample-time' + +RUN: not %Benchmark_O --num-iters \ +RUN: 2>&1 | %FileCheck %s --check-prefix NUMITERS +NUMITERS: error: missing value for '--num-iters' + +RUN: not %Benchmark_O --num-samples \ +RUN: 2>&1 | %FileCheck %s --check-prefix NUMSAMPLES +NUMSAMPLES: error: missing value for '--num-samples' + +RUN: not %Benchmark_O --sleep \ +RUN: 2>&1 | %FileCheck %s --check-prefix SLEEP +SLEEP: error: missing value for '--sleep' + +RUN: not %Benchmark_O --delim \ +RUN: 2>&1 | %FileCheck %s --check-prefix DELIM +DELIM: error: missing value for '--delim' + +RUN: not %Benchmark_O --tags=bogus \ +RUN: 2>&1 | %FileCheck %s --check-prefix BADTAG +BADTAG: error: 'bogus' is not a valid 'BenchmarkCategory' + +RUN: not %Benchmark_O --skip-tags=bogus \ +RUN: 2>&1 | %FileCheck %s --check-prefix BADSKIPTAG +BADSKIPTAG: error: 'bogus' is not a valid 'BenchmarkCategory' + +```` + +## Usage + +```` +RUN: %Benchmark_O --help | %FileCheck %s --check-prefix OPTIONS +OPTIONS: usage: Benchmark_O [--argument=VALUE] [TEST [TEST ...]] +OPTIONS: optional arguments: +OPTIONS: --help +OPTIONS-SAME: show this help message and exit +OPTIONS: --verbose +OPTIONS: --delim +OPTIONS: --tags +OPTIONS: --list +```` diff --git a/test/benchmark/lit.local.cfg b/test/benchmark/lit.local.cfg new file mode 100644 index 0000000000000..614deea3f910a --- /dev/null +++ b/test/benchmark/lit.local.cfg @@ -0,0 +1,2 @@ +# suffixes: A list of file extensions to treat as test files. +config.suffixes.add('.md')