From 2ec19ecb4608520eec0ac21b2594f65fad788a46 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Tue, 26 Aug 2025 16:15:37 -0400 Subject: [PATCH 1/3] Benchmarks: Skip long benchmarks in -Onone build --- benchmark/cmake/modules/AddSwiftBenchmarkSuite.cmake | 8 ++++++++ benchmark/multi-source/Monoids/Benchmark.swift | 2 +- benchmark/scripts/build_script_helper.py | 9 ++++++++- benchmark/utils/DriverUtils.swift | 8 +++++++- benchmark/utils/TestsUtils.swift | 3 +++ 5 files changed, 27 insertions(+), 3 deletions(-) diff --git a/benchmark/cmake/modules/AddSwiftBenchmarkSuite.cmake b/benchmark/cmake/modules/AddSwiftBenchmarkSuite.cmake index 2a542c9cd2291..d7b7803adcee2 100644 --- a/benchmark/cmake/modules/AddSwiftBenchmarkSuite.cmake +++ b/benchmark/cmake/modules/AddSwiftBenchmarkSuite.cmake @@ -363,6 +363,10 @@ function (swift_benchmark_compile_archopts) list(APPEND common_options "-g") endif() + if("${optflag}" STREQUAL "Onone") + list(APPEND common_options "-DDEBUG") + endif() + if (is_darwin) list(APPEND common_options "-I" "${srcdir}/utils/ObjectiveCTests" @@ -400,6 +404,10 @@ function (swift_benchmark_compile_archopts) "-target" "${target}" "-${driver_opt}") + if(${optflag} STREQUAL "Onone") + list(APPEND common_options_driver "-DDEBUG") + endif() + if(SWIFT_BENCHMARK_GENERATE_DEBUG_INFO) list(APPEND common_options_driver "-g") endif() diff --git a/benchmark/multi-source/Monoids/Benchmark.swift b/benchmark/multi-source/Monoids/Benchmark.swift index 87fda858e2920..9907fe342e0bc 100644 --- a/benchmark/multi-source/Monoids/Benchmark.swift +++ b/benchmark/multi-source/Monoids/Benchmark.swift @@ -5,7 +5,7 @@ public let benchmarks = [ BenchmarkInfo( name: "Monoids", runFunction: run_Monoids, - tags: [.algorithm]) + tags: [.algorithm, .miniapplication, .long]) ] func run_Monoids(_ n: Int) { diff --git a/benchmark/scripts/build_script_helper.py b/benchmark/scripts/build_script_helper.py index cfec15145ac1d..82feb524cd630 100755 --- a/benchmark/scripts/build_script_helper.py +++ b/benchmark/scripts/build_script_helper.py @@ -25,8 +25,15 @@ def perform_build(args, swiftbuild_path, config, binary_name, opt_flag): "-Xswiftc", "-align-module-to-page-size", "-Xswiftc", - opt_flag, + opt_flag ] + + if config == "debug": + swiftbuild_args += [ + "-Xswiftc", + "-DDEBUG" + ] + if args.verbose: swiftbuild_args.append("--verbose") subprocess.call(swiftbuild_args) diff --git a/benchmark/utils/DriverUtils.swift b/benchmark/utils/DriverUtils.swift index 733c022b60d83..8b41c8311a04b 100644 --- a/benchmark/utils/DriverUtils.swift +++ b/benchmark/utils/DriverUtils.swift @@ -198,10 +198,16 @@ struct TestConfig { action = c.action ?? .run allowNondeterministicHashing = c.allowNondeterministicHashing ?? false jsonOutput = c.jsonOutput ?? false + + var skipTags: Set + skipTags = c.tags ?? [.unstable, .skip] +#if DEBUG + skipTags.insert(.long) +#endif tests = TestConfig.filterTests(registeredBenchmarks, tests: c.tests ?? [], tags: c.tags ?? [], - skipTags: c.skipTags ?? [.unstable, .skip]) + skipTags: skipTags) if tests.count > 0 { testNameLength = tests.map{$0.info.name.count}.sorted().reversed().first! diff --git a/benchmark/utils/TestsUtils.swift b/benchmark/utils/TestsUtils.swift index 07d9e35c3359d..1289a7f23d886 100644 --- a/benchmark/utils/TestsUtils.swift +++ b/benchmark/utils/TestsUtils.swift @@ -69,6 +69,9 @@ public enum BenchmarkCategory : String { // significant optimization. case cpubench + // Benchmarks to skip on -Onone runs. + case long + // Explicit skip marker case skip } From 48eddac961d69484abef35cefc829e0481d8cf48 Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Sat, 23 Aug 2025 14:35:06 -0400 Subject: [PATCH 2/3] Benchmarks: Add support for async benchmarks --- benchmark/utils/DriverUtils.swift | 40 ++++++++++++++++++------------- benchmark/utils/TestsUtils.swift | 6 ++--- benchmark/utils/main.swift | 2 +- 3 files changed, 27 insertions(+), 21 deletions(-) diff --git a/benchmark/utils/DriverUtils.swift b/benchmark/utils/DriverUtils.swift index 8b41c8311a04b..5785a10dc8174 100644 --- a/benchmark/utils/DriverUtils.swift +++ b/benchmark/utils/DriverUtils.swift @@ -23,6 +23,12 @@ import LibProc import TestsUtils +/// Sorry. +private func ??(_ x: T?, _ y: @autoclosure () async -> T) async -> T { + if let x { return x } + return await y() +} + struct MeasurementMetadata { // Note: maxRSS and pages subtract the RSS measured // after the benchmark driver setup has finished. @@ -487,13 +493,13 @@ final class TestRunner { } /// Measure the `fn` and return the average sample time per iteration (μs). - func measure(_ name: String, fn: (Int) -> Void, numIters: Int) -> Double { + func measure(_ name: String, fn: (Int) async -> Void, numIters: Int) async -> Double { #if SWIFT_RUNTIME_ENABLE_LEAK_CHECKER name.withCString { p in startTrackingObjects(p) } #endif startMeasurement() - fn(numIters) + await fn(numIters) stopMeasurement() #if SWIFT_RUNTIME_ENABLE_LEAK_CHECKER @@ -508,7 +514,7 @@ final class TestRunner { } /// Run the benchmark and return the measured results. - func run(_ test: BenchmarkInfo) -> BenchResults? { + func run(_ test: BenchmarkInfo) async -> BenchResults? { // Before we do anything, check that we actually have a function to // run. If we don't it is because the benchmark is not supported on // the platform and we should skip it. @@ -534,8 +540,8 @@ final class TestRunner { } // Determine number of iterations for testFn to run for desired time. - func iterationsPerSampleTime() -> (numIters: Int, oneIter: Double) { - let oneIter = measure(test.name, fn: testFn, numIters: 1) + func iterationsPerSampleTime() async -> (numIters: Int, oneIter: Double) { + let oneIter = await measure(test.name, fn: testFn, numIters: 1) if oneIter > 0 { let timePerSample = c.sampleTime * 1_000_000.0 // microseconds (μs) return (max(Int(timePerSample / oneIter), 1), oneIter) @@ -546,8 +552,8 @@ final class TestRunner { // Determine the scale of measurements. Re-use the calibration result if // it is just one measurement. - func calibrateMeasurements() -> Int { - let (numIters, oneIter) = iterationsPerSampleTime() + func calibrateMeasurements() async -> Int { + let (numIters, oneIter) = await iterationsPerSampleTime() if numIters == 1 { addSample(oneIter) } else { resetMeasurements() } // for accurate yielding reports return numIters @@ -555,19 +561,19 @@ final class TestRunner { let numIters = min( // Cap to prevent overflow on 32-bit systems when scaled Int.max / 10_000, // by the inner loop multiplier inside the `testFn`. - c.numIters ?? calibrateMeasurements()) + await c.numIters ?? (await calibrateMeasurements())) - let numSamples = c.numSamples ?? + let numSamples = await c.numSamples ?? // Compute the number of samples to measure for `sample-time`, // clamped in (`min-samples`, 200) range, if the `num-iters` are fixed. - max(c.minSamples ?? 1, min(200, c.numIters == nil ? 1 : - calibrateMeasurements())) + (max(await c.minSamples ?? 1, min(200, c.numIters == nil ? 1 : + await calibrateMeasurements()))) samples.reserveCapacity(numSamples) logVerbose(" Collecting \(numSamples) samples.") logVerbose(" Measuring with scale \(numIters).") for _ in samples.count.. () + private var _runFunction: (Int) async -> () /// A function that invokes the specific benchmark routine. - public var runFunction: ((Int) -> ())? { + public var runFunction: ((Int) async -> ())? { if !shouldRun { return nil } @@ -174,7 +174,7 @@ public struct BenchmarkInfo { /// to be interrupted by a context switch. public var legacyFactor: Int? - public init(name: String, runFunction: @escaping (Int) -> (), tags: [BenchmarkCategory], + public init(name: String, runFunction: @escaping (Int) async -> (), tags: [BenchmarkCategory], setUpFunction: (() -> ())? = nil, tearDownFunction: (() -> ())? = nil, unsupportedPlatforms: BenchmarkPlatformSet = [], diff --git a/benchmark/utils/main.swift b/benchmark/utils/main.swift index 6d866f8092825..f19e3ea8951d8 100644 --- a/benchmark/utils/main.swift +++ b/benchmark/utils/main.swift @@ -421,4 +421,4 @@ register(Walsh.benchmarks) register(WordCount.benchmarks) register(XorLoop.benchmarks) -main() +await main() From d7864301b61bec9fb475d979b5c962c766d0f13c Mon Sep 17 00:00:00 2001 From: Slava Pestov Date: Sat, 23 Aug 2025 14:35:23 -0400 Subject: [PATCH 3/3] Benchmarks: Remove DispatchSemaphore hack from Monoids benchmark --- benchmark/multi-source/Monoids/Benchmark.swift | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/benchmark/multi-source/Monoids/Benchmark.swift b/benchmark/multi-source/Monoids/Benchmark.swift index 9907fe342e0bc..4d391af453f67 100644 --- a/benchmark/multi-source/Monoids/Benchmark.swift +++ b/benchmark/multi-source/Monoids/Benchmark.swift @@ -1,5 +1,4 @@ import TestsUtils -import Dispatch public let benchmarks = [ BenchmarkInfo( @@ -8,13 +7,8 @@ public let benchmarks = [ tags: [.algorithm, .miniapplication, .long]) ] -func run_Monoids(_ n: Int) { - let semaphore = DispatchSemaphore(value: 0) +func run_Monoids(_ n: Int) async { for _ in 0 ... n { - Task { - await run(output: false) - semaphore.signal() - } - semaphore.wait() + await run(output: false) } }