Skip to content

Commit a197f20

Browse files
palantjoseluisq
palant
and
joseluisq
authored
feat: load testing benchmarks comparison for each commit via Github Actions (#355)
It runs load testing benchmarks comparison of SWS against several major web servers via CI regularly. Results can be found at https://github.com/static-web-server/static-web-server/actions/runs/8802645161 and benchmark setup and corpus can be found at https://github.com/static-web-server/benchmarks-setup * Run benchmarks for each commit via Github Actions * Enable compression in all tested software * Make sure lighttpd compresses the static file * Use static config files * Don't repeat vegeta command line parameters * Use gnuplot to produce an overview graph * Use graph-cli instead of gnuplot * Use a dedicated test root directory and run various tests on it * Display performance overview as job summary * Remove scheduled test run * Add (source) suffix to SWS version * Merge artifacts after successful run * Do not upload binary reports * Simplify subdirectory names for merged artifact * Move configuration and test root used for benchmarking into a separate repository --------- Co-authored-by: Jose Quintana <1700322+joseluisq@users.noreply.github.com>
1 parent c3c55a4 commit a197f20

File tree

2 files changed

+213
-0
lines changed

2 files changed

+213
-0
lines changed

‎.github/workflows/perfcheck.yml‎

Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,212 @@
1+
name: perfcheck
2+
on:
3+
pull_request:
4+
paths:
5+
- .github/workflows/perfcheck.yml
6+
- .cargo/config.toml
7+
- scripts/ci/vegeta_report_to_csv.jq
8+
- Cargo.lock
9+
- Cargo.toml
10+
- src/**
11+
push:
12+
branches:
13+
- master
14+
- staging
15+
- trying
16+
paths:
17+
- .github/workflows/perfcheck.yml
18+
- .cargo/config.toml
19+
- scripts/ci/vegeta_report_to_csv.jq
20+
- Cargo.lock
21+
- Cargo.toml
22+
- src/**
23+
24+
jobs:
25+
benchmark:
26+
name: benchmark
27+
runs-on: ubuntu-22.04
28+
env:
29+
# Cargo binary
30+
CARGO_BIN: cargo
31+
# When CARGO_BIN is set to CROSS, this is set to `--target matrix.target`.
32+
TARGET_FLAGS: ""
33+
# When CARGO_BIN is set to CROSS, TARGET_DIR includes matrix.target.
34+
TARGET_DIR: ./target
35+
# Rustc flags needed by the `all` features
36+
RUSTFLAGS: "--cfg tokio_unstable"
37+
# SWS features for Cargo build
38+
CARGO_FEATURES: "--features=all"
39+
# Version of the vegeta binary
40+
VEGETA_VERSION: "12.11.1"
41+
# Command-line parameters for the vegeta binary
42+
VEGETA_FLAGS: "-workers=4 -connections=100 -rate=${{ matrix.rate }} -duration=10s"
43+
44+
strategy:
45+
matrix:
46+
include:
47+
- title: "Small file (dynamic compression)"
48+
url: http://localhost:8080/small.txt
49+
rate: "1000/s"
50+
- title: "Small file (pre-compressed)"
51+
url: http://localhost:8080/small_precompressed.txt
52+
rate: "1000/s"
53+
- title: "Large file (dynamic compression)"
54+
url: http://localhost:8080/large.txt
55+
rate: "200/s"
56+
- title: "Large file (pre-compressed)"
57+
url: http://localhost:8080/large_precompressed.txt
58+
rate: "500/s"
59+
- title: "Directory listing (many files)"
60+
url: http://localhost:8080/
61+
rate: "1000/s"
62+
- title: "Directory listing (empty directory)"
63+
url: http://localhost:8080/emptydir/
64+
rate: "1000/s"
65+
- title: "Small directory index file"
66+
url: http://localhost:8080/dirsmallindex/
67+
rate: "1000/s"
68+
- title: "Large directory index file"
69+
url: http://localhost:8080/dirlargeindex/
70+
rate: "200/s"
71+
- title: "Not found"
72+
url: http://localhost:8080/not_found
73+
rate: "1000/s"
74+
75+
steps:
76+
- name: Checkout repository
77+
uses: actions/checkout@v4
78+
with:
79+
fetch-depth: 1
80+
81+
- name: Checkout benchmark-setup repository
82+
uses: actions/checkout@v4
83+
with:
84+
repository: static-web-server/benchmarks-setup
85+
ref: v0.1
86+
path: benchmarks-setup
87+
fetch-depth: 1
88+
89+
- name: Cache Rust Cargo/Toolchain
90+
uses: actions/cache@v4
91+
with:
92+
path: |
93+
~/.cargo
94+
**/target
95+
key: rust-${{ hashFiles('**/Cargo.lock') }}
96+
restore-keys: |
97+
rust-
98+
99+
- name: Install vegeta binary
100+
run: |
101+
curl -L https://github.com/tsenart/vegeta/releases/download/v${{ env.VEGETA_VERSION }}/vegeta_${{ env.VEGETA_VERSION }}_linux_amd64.tar.gz | tar xz
102+
echo "Downloaded vegeta"
103+
./vegeta --version
104+
105+
- name: Install web server software
106+
uses: awalsh128/cache-apt-pkgs-action@latest
107+
with:
108+
packages: apache2 lighttpd nginx
109+
version: 1.0
110+
111+
- name: Install Rust
112+
uses: dtolnay/rust-toolchain@stable
113+
with:
114+
toolchain: stable
115+
target: x86_64-unknown-linux-gnu
116+
117+
- name: Show command used for Cargo
118+
run: |
119+
echo "cargo command is: ${{ env.CARGO_BIN }}"
120+
echo "target flag is: ${{ env.TARGET_FLAGS }}"
121+
echo "target dir is: ${{ env.TARGET_DIR }}"
122+
123+
- name: Build release binary
124+
run: ${{ env.CARGO_BIN }} build --bin static-web-server -vv --release ${{ env.CARGO_FEATURES }} ${{ env.TARGET_FLAGS }}
125+
126+
- name: Initialize graph.csv
127+
run: echo "Sample, min, mean, 50th, 90th, 95th, 99th, max" > graph.csv
128+
129+
- name: Run SWS benchmark
130+
run: |
131+
NAME=$(${{ env.TARGET_DIR }}/release/static-web-server --version | sed -e 's/static-web-server/sws/' -e 's/$/ (source)/')
132+
REPORT=report_sws.bin
133+
${{ env.TARGET_DIR }}/release/static-web-server --config-file benchmarks-setup/config/sws.toml &
134+
sleep 2
135+
echo "GET ${{ matrix.url }}" | ./vegeta attack -name "$NAME" ${{ env.VEGETA_FLAGS }} > $REPORT
136+
kill %1
137+
./vegeta report -type=text $REPORT
138+
./vegeta report -type=json $REPORT | jq --arg name "$NAME" -r -f scripts/ci/vegeta_report_to_csv.jq >> graph.csv
139+
140+
- name: Run Apache benchmark
141+
run: |
142+
NAME=$(apache2 -v | sed 's/.*: //' | sed 's/ .*//' | sed 's!/! !')
143+
REPORT=report_apache.bin
144+
apache2 -D FOREGROUND -f `pwd`/benchmarks-setup/config/apache.conf &
145+
sleep 2
146+
echo "GET ${{ matrix.url }}" | ./vegeta attack -name "$NAME" ${{ env.VEGETA_FLAGS }} > $REPORT
147+
kill %1
148+
./vegeta report -type=text $REPORT
149+
./vegeta report -type=json $REPORT | jq --arg name "$NAME" -r -f scripts/ci/vegeta_report_to_csv.jq >> graph.csv
150+
151+
- name: Run lighttpd benchmark
152+
run: |
153+
NAME=$(lighttpd -v | sed 's/ .*//' | sed 's!/! !')
154+
REPORT=report_lighttpd.bin
155+
lighttpd -D -f `pwd`/benchmarks-setup/config/lighttpd.conf &
156+
sleep 2
157+
URL="${{ matrix.url }}"
158+
if [[ $URL == *"_precompressed"* ]]; then
159+
# lighttpd does not really support pre-compressed files, request them explicitly
160+
URL="${{ matrix.url }}.gz"
161+
fi
162+
echo "GET $URL" | ./vegeta attack -name "$NAME" ${{ env.VEGETA_FLAGS }} > $REPORT
163+
kill %1
164+
./vegeta report -type=text $REPORT
165+
./vegeta report -type=json $REPORT | jq --arg name "$NAME" -r -f scripts/ci/vegeta_report_to_csv.jq >> graph.csv
166+
167+
- name: Run nginx benchmark
168+
run: |
169+
NAME=$(nginx -v 2>&1 | sed 's/.*: //' | sed 's/ .*//' | sed 's!/! !')
170+
REPORT=report_nginx.bin
171+
nginx -c `pwd`/benchmarks-setup/config/nginx.conf &
172+
sleep 2
173+
echo "GET ${{ matrix.url }}" | ./vegeta attack -name "$NAME" ${{ env.VEGETA_FLAGS }} > $REPORT
174+
kill %1
175+
./vegeta report -type=text $REPORT
176+
./vegeta report -type=json $REPORT | jq --arg name "$NAME" -r -f scripts/ci/vegeta_report_to_csv.jq >> graph.csv
177+
178+
- name: Set up Python
179+
uses: actions/setup-python@v5
180+
with:
181+
python-version: '3.12'
182+
183+
- name: Install pipx
184+
run: pip install pipx
185+
186+
- name: Produce plot
187+
run: |
188+
./vegeta plot --title "${{ matrix.title }}" report_*.bin > detailed_plot.html
189+
cat graph.csv
190+
cat graph.csv | sed -e 's/,/|/g' -e 's/"//g' -e 's/^/|/' -e 's/$/|/' -e '2 i |-|-|-|-|-|-|-|-|' >> $GITHUB_STEP_SUMMARY
191+
pipx run graph-cli --barh --bar-label --xlabel '' --ylabel 'Latency (ms)' --title "${{ matrix.title }}" --figsize 1200x700 --fontsize 10 --ytick-angle 45 --ytick-align top -o overview.png graph.csv
192+
193+
- name: Upload benchmark plots
194+
uses: actions/upload-artifact@v4
195+
with:
196+
name: ${{ matrix.title }}
197+
path: |
198+
detailed_plot.html
199+
overview.png
200+
201+
merge-artifacts:
202+
name: merge-artifacts
203+
needs: benchmark
204+
runs-on: ubuntu-22.04
205+
206+
steps:
207+
- name: Merge benchmark plots
208+
uses: actions/upload-artifact/merge@v4
209+
with:
210+
name: Benchmark plots
211+
separate-directories: true
212+
delete-merged: true

‎scripts/ci/vegeta_report_to_csv.jq‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[$name, (.latencies["min", "mean", "50th", "90th", "95th", "99th", "max"] | . / 1000 | round / 1000)] | @csv

0 commit comments

Comments
 (0)