diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 0335b817..0611eed7 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -67,34 +67,55 @@ jobs: run: | echo '## Summary' >> $GITHUB_STEP_SUMMARY echo $(date) >> $GITHUB_STEP_SUMMARY - echo "exitStatus=1" >> $GITHUB_ENV + # if we had access to fd 3 we could do this in one call eg { var=$(cmd 3>&2 2>&1 1>&3); } 2>&1 + # but unfortunately we don't so we have to run the baseline check twice once to extract stdout + # and once to extract stderr + BENCHMARK_STDERR=$(swift package benchmark baseline check main pull_request 2>&1) swift package benchmark baseline check main pull_request --format markdown >> $GITHUB_STEP_SUMMARY - echo '---' >> $GITHUB_STEP_SUMMARY - swift package benchmark baseline compare main pull_request --no-progress --quiet --format markdown >> $GITHUB_STEP_SUMMARY - echo "exitStatus=0" >> $GITHUB_ENV + echo "exit-status=$?" >> $GITHUB_OUTPUT + echo "benchmark-error=$(echo -e "$BENCHMARK_STDERR" | grep -e "^error: .*" | tail -n 1 | cut -c 8-)" >> $GITHUB_OUTPUT continue-on-error: true - - if: ${{ env.exitStatus == '0' }} - name: Pull request comment text success - id: prtestsuccess + - name: Pull request comment text + id: benchmark-comment run: | - echo 'PRTEST<> $GITHUB_ENV - echo "[Pull request benchmark comparison [${{ matrix.os }}] with 'main' run at $(date -Iseconds)](https://github.com/adam-fowler/${{ github.event.repository.name }}/actions/runs/${{ github.run_id }})" >> $GITHUB_ENV - echo 'EOF' >> $GITHUB_ENV - - if: ${{ env.exitStatus == '1' }} - name: Pull request comment text failure - id: prtestfailure - run: | - echo 'PRTEST<> $GITHUB_ENV - echo "[Pull request benchmark comparison [${{ matrix.os }}] with 'main' run at $(date -Iseconds)](https://github.com/adam-fowler/${{ github.event.repository.name }}/actions/runs/${{ github.run_id }})" >> $GITHUB_ENV - echo "_Pull request had performance regressions_" >> $GITHUB_ENV - echo 'EOF' >> $GITHUB_ENV + echo "[Pull request benchmark comparison [${{ matrix.os }}] with 'main' run at $(date -Iseconds)](https://github.com/adam-fowler/${{ github.event.repository.name }}/actions/runs/${{ github.run_id }})" >> comment.md + + EXIT_CODE='${{steps.benchmark.outputs.exit-status}}' + + case "${EXIT_CODE}" in + 0) + echo "_Pull request no significant performance differences ✅_" >> comment.md + echo "exitStatus=0" >> $GITHUB_ENV + ;; + *) + # Get error string from benchmark output + BENCHMARK_ERROR='${{steps.benchmark.outputs.benchmark-error}}' + case "${BENCHMARK_ERROR}" in + "benchmarkThresholdRegression") + echo "_Pull request has performance regressions ❌_" >> comment.md + echo "exitStatus=1" >> $GITHUB_ENV + ;; + "benchmarkThresholdImprovement") + echo "_Pull request has performance improvements ✅_" >> comment.md + echo "exitStatus=0" >> $GITHUB_ENV + ;; + *) + echo "_Benchmark comparison failed with error $BENCHMARK_ERROR ❌_" >> comment.md + echo "exitStatus=1" >> $GITHUB_ENV + ;; + esac + ;; + esac + + echo '---' >> $GITHUB_STEP_SUMMARY + swift package benchmark baseline compare main pull_request --no-progress --quiet --format markdown >> $GITHUB_STEP_SUMMARY - name: Comment PR if: ${{ env.hasBenchmark == '1' }} uses: thollander/actions-comment-pull-request@v3 with: github-token: ${{ secrets.GITHUB_TOKEN }} - message: ${{ env.PRTEST }} + file-path: comment.md comment-tag: benchmark - name: Exit with correct status run: | - exit ${{ env.exitStatus }} + exit ${{env.exitStatus}} diff --git a/.gitignore b/.gitignore index 09bb5f89..791f25fd 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ DerivedData/ .netrc .vscode Package.resolved -.benchmarkBaselines/ \ No newline at end of file +.benchmarkBaselines/ +.swift-version \ No newline at end of file diff --git a/Benchmarks/ValkeyBenchmarks/ValkeyBenchmarks.swift b/Benchmarks/ValkeyBenchmarks/ValkeyBenchmarks.swift index ca05e0bd..5ff00d0c 100644 --- a/Benchmarks/ValkeyBenchmarks/ValkeyBenchmarks.swift +++ b/Benchmarks/ValkeyBenchmarks/ValkeyBenchmarks.swift @@ -24,15 +24,15 @@ let benchmarks: @Sendable () -> Void = { // There is no point comparing wallClock, cpuTotal or throughput on CI as they are too inconsistent ProcessInfo.processInfo.environment["CI"] != nil ? [ - .mallocCountTotal, .instructions, + .mallocCountTotal, ] : [ .wallClock, .cpuTotal, + .instructions, .mallocCountTotal, .throughput, - .instructions, ] var server: Channel?