Skip to content

Commit

Permalink
Refactor Performance Testing (#3953)
Browse files Browse the repository at this point in the history
  • Loading branch information
@nibanks
nibanks committed Dec 14, 2023
1 parent df69a24 commit fe9c297
Show file tree
Hide file tree
Showing 61 changed files with 2,278 additions and 3,840 deletions.
174 changes: 147 additions & 27 deletions scripts/RemoteTests.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
"Tls": ["schannel", "openssl", "openssl3"],
"Arch": ["x64", "x86", "arm", "arm64"],
"Exe": "secnetperf",
"Arguments": "-exec:maxtput -test:Throughput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -uni:1 -timed:1 -upload:12000 -stats:1 -sstats:1"
"Arguments": "-exec:maxtput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -up:12000 -timed:1 -ptput:1 -pconn:1 -pstream:1 -tcp:0"
},
"Variables": [
{
Expand Down Expand Up @@ -46,11 +46,10 @@
"Tls": ["schannel", "openssl", "openssl3"],
"Arch": ["x64", "x86", "arm", "arm64"],
"Exe": "secnetperf",
"Arguments": "-exec:maxtput -test:Throughput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -uni:1 -timed:1 -upload:12000 -tcp:1"
"Arguments": "-exec:maxtput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -up:12000 -timed:1 -ptput:1 -pconn:1 -pstream:1 -tcp:1"
},
"Variables": [
],
"SkipKernel": true,
"AllowLoopback": true,
"Iterations": 5,
"RemoteReadyMatcher": "Started!",
Expand All @@ -66,7 +65,7 @@
"Tls": ["openssl", "openssl3"],
"Arch": ["x64", "arm"],
"Exe": "secnetperf",
"Arguments": "-exec:maxtput -test:Throughput -target:$RemoteAddress -uni:1 -timed:1 -upload:12000 -stats:1 -sstats:1"
"Arguments": "-exec:maxtput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -up:12000 -timed:1 -ptput:1 -pconn:1 -pstream:1 -tcp:0"
},
"Variables": [
{
Expand Down Expand Up @@ -100,11 +99,10 @@
"Tls": ["openssl", "openssl3"],
"Arch": ["x64", "arm"],
"Exe": "secnetperf",
"Arguments": "-exec:maxtput -test:Throughput -target:$RemoteAddress -uni:1 -timed:1 -upload:12000 -tcp:1"
"Arguments": "-exec:maxtput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -up:12000 -timed:1 -ptput:1 -pconn:1 -pstream:1 -tcp:1"
},
"Variables": [
],
"SkipKernel": true,
"AllowLoopback": true,
"Iterations": 5,
"RemoteReadyMatcher": "Started!",
Expand All @@ -120,7 +118,7 @@
"Tls": ["schannel", "openssl", "openssl3"],
"Arch": ["x64", "x86", "arm", "arm64"],
"Exe": "secnetperf",
"Arguments": "-exec:maxtput -test:Throughput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -uni:1 -timed:1 -download:12000 -stats:1"
"Arguments": "-exec:maxtput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -down:12000 -timed:1 -ptput:1 -pconn:1 -tcp:0"
},
"Variables": [
{
Expand All @@ -146,11 +144,10 @@
"Tls": ["schannel", "openssl", "openssl3"],
"Arch": ["x64", "x86", "arm", "arm64"],
"Exe": "secnetperf",
"Arguments": "-exec:maxtput -test:Throughput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -uni:1 -timed:1 -download:12000 -tcp:1"
"Arguments": "-exec:maxtput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -down:12000 -timed:1 -ptput:1 -pconn:1 -tcp:1"
},
"Variables": [
],
"SkipKernel": true,
"AllowLoopback": true,
"Iterations": 5,
"RemoteReadyMatcher": "Started!",
Expand All @@ -166,7 +163,7 @@
"Tls": ["openssl", "openssl3"],
"Arch": ["x64", "arm"],
"Exe": "secnetperf",
"Arguments": "-exec:maxtput -test:Throughput -target:$RemoteAddress -uni:1 -timed:1 -download:12000 -stats:1"
"Arguments": "-exec:maxtput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -down:12000 -timed:1 -ptput:1 -pconn:1 -tcp:0"
},
"Variables": [
{
Expand All @@ -192,11 +189,10 @@
"Tls": ["openssl", "openssl3"],
"Arch": ["x64", "arm"],
"Exe": "secnetperf",
"Arguments": "-exec:maxtput -test:Throughput -target:$RemoteAddress -uni:1 -timed:1 -download:12000 -tcp:1"
"Arguments": "-exec:maxtput -target:$RemoteAddress -bind:$LocalAddress:4434 -ip:4 -down:12000 -timed:1 -ptput:1 -pconn:1 -tcp:1"
},
"Variables": [
],
"SkipKernel": true,
"AllowLoopback": true,
"Iterations": 5,
"RemoteReadyMatcher": "Started!",
Expand All @@ -212,16 +208,16 @@
"Tls": ["schannel", "openssl", "openssl3"],
"Arch": ["x64", "x86", "arm", "arm64"],
"Exe": "secnetperf",
"Arguments": "-exec:lowlat -test:RPS -target:$RemoteAddress"
"Arguments": "-exec:lowlat -target:$RemoteAddress -rstream:1 -plat:1 -tcp:0"
},
"Variables": [
{
"Name": "ConnectionCount",
"Local": {
"1": "-conns:1 -requests:1 -runtime:30000",
"40": "-conns:40 -requests:20",
"250": "-conns:250 -requests:7500",
"1000": "-conns:1 -requests:1000"
"40": "-conns:40 -requests:1 -runtime:10000",
"250": "-conns:250 -requests:30 -runtime:10000",
"1000": "-conns:1 -requests:1000 -runtime:10000"
},
"Default": "250"
},
Expand All @@ -246,8 +242,53 @@
"AllowLoopback": false,
"Iterations": 5,
"RemoteReadyMatcher": "Started!",
"ResultsMatcher": "Result: (.*) RPS, Min: (.*), Max: (.*), 50th: (.*), 90th: (.*), 99th: (.*), 99.9th: (.*), 99.99th: (.*), 99.999th: (.*), 99.9999th: (.*), StdErr: (.*)",
"Formats": ["{0} RPS", "Minimum: {0}", "Maximum: {0}", "Percentiles: 50th: {0}", "90th: {0}", "99th: {0}", "99.9th: {0}", "99.99th: {0}", "99.999th: {0}", "99.9999th: {0}", "Standard Error: {0}"],
"ResultsMatcher": "Result: (.*) RPS, Latency,us 0th: (.*), 50th: (.*), 90th: (.*), 99th: (.*), 99.9th: (.*), 99.99th: (.*), 99.999th: (.*), 99.9999th: (.*), Max: (.*)",
"Formats": ["{0} RPS", "Minimum: {0}", "50th: {0}", "90th: {0}", "99th: {0}", "99.9th: {0}", "99.99th: {0}", "99.999th: {0}", "99.9999th: {0}", "Maximum: {0}"],
"RegressionThreshold": "-50.0"
},
{
"TestName": "TcpRPS",
"Local": {
"Platform": "Windows",
"Tls": ["schannel", "openssl", "openssl3"],
"Arch": ["x64", "x86", "arm", "arm64"],
"Exe": "secnetperf",
"Arguments": "-exec:lowlat -target:$RemoteAddress -rstream:1 -plat:1 -tcp:1"
},
"Variables": [
{
"Name": "ConnectionCount",
"Local": {
"1": "-conns:1 -requests:1 -runtime:30000",
"40": "-conns:40 -requests:1 -runtime:10000",
"250": "-conns:250 -requests:30 -runtime:10000",
"1000": "-conns:1 -requests:1000 -runtime:10000"
},
"Default": "250"
},
{
"Name": "RequestSize",
"Local": {
"0": "-request:0"
},
"Default": "0"
},
{
"Name": "ResponseSize",
"Local": {
"0": "-response:0",
"512": "-response:512",
"4096": "-response:4096",
"16384": "-response:16384"
},
"Default": "4096"
}
],
"AllowLoopback": false,
"Iterations": 5,
"RemoteReadyMatcher": "Started!",
"ResultsMatcher": "Result: (.*) RPS, Latency,us 0th: (.*), 50th: (.*), 90th: (.*), 99th: (.*), 99.9th: (.*), 99.99th: (.*), 99.999th: (.*), 99.9999th: (.*), Max: (.*)",
"Formats": ["{0} RPS", "Minimum: {0}", "50th: {0}", "90th: {0}", "99th: {0}", "99.9th: {0}", "99.99th: {0}", "99.999th: {0}", "99.9999th: {0}", "Maximum: {0}"],
"RegressionThreshold": "-50.0"
},
{
Expand All @@ -257,16 +298,16 @@
"Tls": ["openssl", "openssl3"],
"Arch": ["x64", "arm"],
"Exe": "secnetperf",
"Arguments": "-exec:lowlat -test:RPS -target:$RemoteAddress"
"Arguments": "-exec:lowlat -target:$RemoteAddress -rstream:1 -plat:1 -tcp:0"
},
"Variables": [
{
"Name": "ConnectionCount",
"Local": {
"1": "-conns:1 -requests:1",
"40": "-conns:40 -requests:20",
"250": "-conns:250 -requests:7500",
"1000": "-conns:1 -requests:1000"
"1": "-conns:1 -requests:1 -runtime:30000",
"40": "-conns:40 -requests:1 -runtime:10000",
"250": "-conns:250 -requests:30 -runtime:10000",
"1000": "-conns:1 -requests:1000 -runtime:10000"
},
"Default": "250"
},
Expand All @@ -291,8 +332,53 @@
"AllowLoopback": false,
"Iterations": 5,
"RemoteReadyMatcher": "Started!",
"ResultsMatcher": "Result: (.*) RPS, Min: (.*), Max: (.*), 50th: (.*), 90th: (.*), 99th: (.*), 99.9th: (.*), 99.99th: (.*), 99.999th: (.*), 99.9999th: (.*), StdErr: (.*)",
"Formats": ["{0} RPS", "Minimum: {0}", "Maximum: {0}", "Percentiles: 50th: {0}", "90th: {0}", "99th: {0}", "99.9th: {0}", "99.99th: {0}", "99.999th: {0}", "99.9999th: {0}", "Standard Error: {0}"],
"ResultsMatcher": "Result: (.*) RPS, Latency,us 0th: (.*), 50th: (.*), 90th: (.*), 99th: (.*), 99.9th: (.*), 99.99th: (.*), 99.999th: (.*), 99.9999th: (.*), Max: (.*)",
"Formats": ["{0} RPS", "Minimum: {0}", "50th: {0}", "90th: {0}", "99th: {0}", "99.9th: {0}", "99.99th: {0}", "99.999th: {0}", "99.9999th: {0}", "Maximum: {0}"],
"RegressionThreshold": "-75.0"
},
{
"TestName": "TcpRPS",
"Local": {
"Platform": "linux",
"Tls": ["openssl", "openssl3"],
"Arch": ["x64", "arm"],
"Exe": "secnetperf",
"Arguments": "-exec:lowlat -target:$RemoteAddress -rstream:1 -plat:1 -tcp:1"
},
"Variables": [
{
"Name": "ConnectionCount",
"Local": {
"1": "-conns:1 -requests:1 -runtime:30000",
"40": "-conns:40 -requests:1 -runtime:10000",
"250": "-conns:250 -requests:30 -runtime:10000",
"1000": "-conns:1 -requests:1000 -runtime:10000"
},
"Default": "250"
},
{
"Name": "RequestSize",
"Local": {
"0": "-request:0"
},
"Default": "0"
},
{
"Name": "ResponseSize",
"Local": {
"0": "-response:0",
"512": "-response:512",
"4096": "-response:4096",
"16384": "-response:16384"
},
"Default": "4096"
}
],
"AllowLoopback": false,
"Iterations": 5,
"RemoteReadyMatcher": "Started!",
"ResultsMatcher": "Result: (.*) RPS, Latency,us 0th: (.*), 50th: (.*), 90th: (.*), 99th: (.*), 99.9th: (.*), 99.99th: (.*), 99.999th: (.*), 99.9999th: (.*), Max: (.*)",
"Formats": ["{0} RPS", "Minimum: {0}", "50th: {0}", "90th: {0}", "99th: {0}", "99.9th: {0}", "99.99th: {0}", "99.999th: {0}", "99.9999th: {0}", "Maximum: {0}"],
"RegressionThreshold": "-75.0"
},
{
Expand All @@ -302,7 +388,24 @@
"Tls": ["schannel", "openssl", "openssl3"],
"Arch": ["x64", "x86", "arm", "arm64"],
"Exe": "secnetperf",
"Arguments": "-exec:maxtput -test:HPS -target:$RemoteAddress -incrementtarget:1"
"Arguments": "-exec:maxtput -target:$RemoteAddress -affinitize:1 -inctarget:1 -rconn:1 -conns:800 -share:1 -run:10000 -tcp:0 -prate:1"
},
"Variables": [],
"AllowLoopback": false,
"Iterations": 5,
"RemoteReadyMatcher": "Started!",
"ResultsMatcher": "Result: (.*) HPS.*",
"Formats": ["{0} HPS"],
"RegressionThreshold": "-40.0"
},
{
"TestName": "TcpHPS",
"Local": {
"Platform": "Windows",
"Tls": ["schannel", "openssl", "openssl3"],
"Arch": ["x64", "x86", "arm", "arm64"],
"Exe": "secnetperf",
"Arguments": "-exec:maxtput -target:$RemoteAddress -affinitize:1 -inctarget:1 -rconn:1 -conns:800 -share:1 -run:10000 -tcp:1 -prate:1"
},
"Variables": [],
"AllowLoopback": false,
Expand All @@ -319,7 +422,24 @@
"Tls": ["openssl", "openssl3"],
"Arch": ["x64", "arm"],
"Exe": "secnetperf",
"Arguments": "-exec:maxtput -test:HPS -target:$RemoteAddress"
"Arguments": "-exec:maxtput -target:$RemoteAddress -affinitize:1 -inctarget:1 -rconn:1 -conns:800 -share:1 -run:10000 -tcp:0 -prate:1"
},
"Variables": [],
"AllowLoopback": false,
"Iterations": 5,
"RemoteReadyMatcher": "Started!",
"ResultsMatcher": "Result: (.*) HPS.*",
"Formats": ["{0} HPS"],
"RegressionThreshold": "-75.0"
},
{
"TestName": "TcpHPS",
"Local": {
"Platform": "linux",
"Tls": ["openssl", "openssl3"],
"Arch": ["x64", "arm"],
"Exe": "secnetperf",
"Arguments": "-exec:maxtput -target:$RemoteAddress -affinitize:1 -inctarget:1 -rconn:1 -conns:800 -share:1 -run:10000 -tcp:1 -prate:1"
},
"Variables": [],
"AllowLoopback": false,
Expand Down
15 changes: 1 addition & 14 deletions scripts/clog.inputs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,22 +95,10 @@
../src/test/lib/QuicDrill.cpp
../src/test/lib/DatagramTest.cpp
../src/test/lib/OwnershipTest.cpp
../src/perf/lib/PerfHelpers.h
../src/perf/lib/PerfClient.cpp
../src/perf/lib/PerfServer.cpp
../src/perf/lib/SecNetPerfMain.cpp
../src/perf/lib/HpsClient.h
../src/perf/lib/PerfServer.h
../src/perf/lib/CMakeLists.txt
../src/perf/lib/Tcp.h
../src/perf/lib/RpsClient.h
../src/perf/lib/Tcp.cpp
../src/perf/lib/ThroughputClient.h
../src/perf/lib/RpsClient.cpp
../src/perf/lib/PerfCommon.h
../src/perf/lib/ThroughputClient.cpp
../src/perf/lib/PerfBase.h
../src/perf/lib/HpsClient.cpp
../src/perf/lib/LatencyHelpers.h
../src/core/unittest/SettingsTest.cpp
../src/core/unittest/SpinFrame.cpp
../src/core/unittest/SlidingWindowExtremumTest.cpp
Expand All @@ -134,6 +122,5 @@
../src/perf/bin/histogram/hdr_histogram.h
../src/perf/bin/histogram/hdr_histogram.c
../src/perf/bin/drvmain.cpp
../src/perf/bin/appmain.cpp
../src/tools/interop/interop.cpp
../src/tools/interop/interop.h
Loading

0 comments on commit fe9c297

Please sign in to comment.