diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index cd669b49..16352631 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -19,3 +19,61 @@ jobs: license_header_check_project_name: "Swift.org" unacceptable_language_check_enabled: false format_check_enabled: false + bench: + name: Benchmark + runs-on: ubuntu-latest + env: + BUILD_CMD: swift build -c release + BENCH_CMD: .build/release/RegexBenchmark + BASELINE_FILE: benchmark-baseline + COMPARE_FILE: benchmark-pr + COMPARE_OUT_FILE: benchmark-results.txt + steps: + - name: Check out baseline branch + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.sha }} + path: base + fetch-depth: 0 + - name: Build baseline branch + working-directory: base + run: | + set -euo pipefail + eval "$BUILD_CMD" + - name: Run baseline benchmark + working-directory: base + run: | + set -euo pipefail + eval "$BENCH_CMD --save $RUNNER_TEMP/$BASELINE_FILE" + test -s "$RUNNER_TEMP/$BASELINE_FILE" || { echo "Baseline not created at $BASELINE_FILE"; exit 1; } + - name: Check out PR branch + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + path: pr + fetch-depth: 0 + - name: Build PR branch + working-directory: pr + run: | + set -euo pipefail + eval "$BUILD_CMD" + - name: Run PR benchmark + working-directory: pr + run: | + set -euo pipefail + eval "$BENCH_CMD --save $RUNNER_TEMP/$COMPARE_FILE" + test -s "$RUNNER_TEMP/$COMPARE_FILE" || { echo "Comparison not created at $COMPARE_FILE"; exit 1; } + eval "$BENCH_CMD --compare $RUNNER_TEMP/$BASELINE_FILE" | tee "$RUNNER_TEMP/$COMPARE_OUT_FILE" + - name: 📊 Compare benchmarks + working-directory: pr + run: | + set -euo pipefail + eval "$BENCH_CMD --load $RUNNER_TEMP/$COMPARE_FILE --compare $RUNNER_TEMP/$BASELINE_FILE --compare-compile-time $RUNNER_TEMP/$BASELINE_FILE" | tee "$RUNNER_TEMP/$COMPARE_OUT_FILE" + - name: Upload benchmark artifacts + uses: actions/upload-artifact@v4 + with: + name: benchmark-results + path: | + ${{ runner.temp }}/${{ env.BASELINE_FILE }} + ${{ runner.temp }}/${{ env.COMPARE_FILE }} + ${{ runner.temp }}/${{ env.COMPARE_OUT_FILE }} diff --git a/Sources/RegexBenchmark/BenchmarkResults.swift b/Sources/RegexBenchmark/BenchmarkResults.swift index 82432230..02eba5a1 100644 --- a/Sources/RegexBenchmark/BenchmarkResults.swift +++ b/Sources/RegexBenchmark/BenchmarkResults.swift @@ -115,12 +115,12 @@ extension BenchmarkRunner { .sorted(by: {(a,b) in a.diff!.seconds < b.diff!.seconds}) print("Comparing against \(against)") - print("=== Regressions ======================================================================") + print("=== Regressions ================================================================") for item in regressions { print(item) } - print("=== Improvements =====================================================================") + print("=== Improvements ===============================================================") for item in improvements { print(item) } @@ -128,7 +128,7 @@ extension BenchmarkRunner { #if os(macOS) && canImport(Charts) if showChart { print(""" - === Comparison chart ================================================================= + === Comparison chart =========================================================== Press Control-C to close... """) BenchmarkResultApp.comparisons = comparisons @@ -234,9 +234,17 @@ extension BenchmarkResult { return "- \(name) N/A" } let percentage = (1000 * diff.seconds / baselineTime.seconds).rounded()/10 - let len = max(40 - name.count, 1) - let nameSpacing = String(repeating: " ", count: len) - return "- \(name)\(nameSpacing)\(latestTime)\t\(baselineTime)\t\(diff)\t\t\(percentage)%" + let start = if name.count > 40 { + "- \(name)\n" + String(repeating: " ", count: 43) + } else { + "- \(name, paddingTo: 40) " + } + return start + """ + \(latestTime, paddingTo: 8, alignRight: true) \ + \(baselineTime, paddingTo: 8, alignRight: true) \ + \(diff, paddingTo: 8, alignRight: true) \ + \(percentage, paddingTo: 5, alignRight: true)% + """ } var asCsv: String { @@ -334,3 +342,16 @@ extension SuiteResult: Codable { return try decoder.decode(SuiteResult.self, from: data) } } + +extension DefaultStringInterpolation { + mutating func appendInterpolation(_ value: T, paddingTo length: Int, alignRight: Bool = false) { + let s = String(describing: value) + let paddingCount = max(0, length - s.count) + let padding = String(repeating: " ", count: paddingCount) + if alignRight { + appendLiteral(padding + s) + } else { + appendLiteral(s + padding) + } + } +} diff --git a/Sources/RegexBenchmark/Utils/Stats.swift b/Sources/RegexBenchmark/Utils/Stats.swift index bc1490d8..8200adca 100644 --- a/Sources/RegexBenchmark/Utils/Stats.swift +++ b/Sources/RegexBenchmark/Utils/Stats.swift @@ -15,7 +15,7 @@ enum Stats {} extension Stats { // Maximum allowed standard deviation is 7.5% of the median runtime - static let maxAllowedStdev = 0.075 + static let maxAllowedStdev = 0.15 static func tTest(_ a: Measurement, _ b: Measurement) -> Bool { // Student's t-test