diff --git a/src/VirtualClient/Scripts/Kusto/Clusters/WorkloadPerformance/Functions/GetMetricsDescriptors.csl b/src/VirtualClient/Scripts/Kusto/Clusters/WorkloadPerformance/Functions/GetMetricsDescriptors.csl index d87e69fd1b..665f512324 100644 --- a/src/VirtualClient/Scripts/Kusto/Clusters/WorkloadPerformance/Functions/GetMetricsDescriptors.csl +++ b/src/VirtualClient/Scripts/Kusto/Clusters/WorkloadPerformance/Functions/GetMetricsDescriptors.csl @@ -31,6 +31,36 @@ "CPS", "SynRttP99", "LowerIsBetter", false, "CPS", "SynRttP99_9", "LowerIsBetter", false, "CPS", "SynRttP99_99", "LowerIsBetter", false, + "NCPS", "RxGbps", "HigherIsBetter", false, + "NCPS", "TxGbps", "HigherIsBetter", false, + "NCPS", "Cps", "HigherIsBetter", false, + "NCPS", "ConnectsPerSec_Min", "HigherIsBetter", false, + "NCPS", "ConnectsPerSec_Max", "HigherIsBetter", false, + "NCPS", "ConnectsPerSec_Med", "HigherIsBetter", false, + "NCPS", "ConnectsPerSec_Avg", "HigherIsBetter", false, + "NCPS", "ConnectsPerSec_P25", "HigherIsBetter", false, + "NCPS", "ConnectsPerSec_P50", "HigherIsBetter", false, + "NCPS", "ConnectsPerSec_P75", "HigherIsBetter", false, + "NCPS", "ConnectsPerSec_P90", "HigherIsBetter", false, + "NCPS", "ConnectsPerSec_P99", "HigherIsBetter", false, + "NCPS", "ConnectsPerSec_P99_9", "HigherIsBetter", false, + "NCPS", "ConnectsPerSec_P99_99", "HigherIsBetter", false, + "NCPS", "ConnectsPerSec_P99_999", "HigherIsBetter", false, + "NCPS", "ConnectsPerSec_Mad", "LowerIsBetter", true, + "NCPS", "ConnectsPerSec_StandardErrorMean", "LowerIsBetter", true, + "NCPS", "ConnectsPerSec_LowerCI", "LowerIsBetter", true, + "NCPS", "ConnectsPerSec_UpperCI", "LowerIsBetter", true, + "NCPS", "RexmitConnPercentage", "LowerIsBetter", false, + "NCPS", "RexmitPerConn", "LowerIsBetter", false, + "NCPS", "SynRttMean", "LowerIsBetter", false, + "NCPS", "SynRttMedian", "LowerIsBetter", false, + "NCPS", "SynRttP25", "LowerIsBetter", false, + "NCPS", "SynRttP75", "LowerIsBetter", false, + "NCPS", "SynRttP90", "LowerIsBetter", false, + "NCPS", "SynRttP95", "LowerIsBetter", false, + "NCPS", "SynRttP99", "LowerIsBetter", false, + "NCPS", "SynRttP99_9", "LowerIsBetter", false, + "NCPS", "SynRttP99_99", "LowerIsBetter", false, "CoreMark", "CoreMark Size", "HigherIsBetter", true, "CoreMark", "Iterations", "HigherIsBetter", false, "CoreMark", "Iterations/Sec", "HigherIsBetter", false, @@ -610,4 +640,4 @@ "SuperBenchmark", "vgg_models/pytorch-vgg19/fp32_train_throughput", "HigherIsBetter", false, "SuperBenchmark", "vgg_models/pytorch-vgg19/return_code:0", "HigherIsBetter", true ] -} \ No newline at end of file +} diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-NETWORK.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-NETWORK.json index 2c1bcee925..dee04fd8c3 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-NETWORK.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-NETWORK.json @@ -8,8 +8,8 @@ }, "Parameters": { "EnableBusyPoll": true, - "CpsPort": 7201, - "CpsDuration": "00:05:00", + "NcpsPort": 9800, + "NcpsDuration": "00:05:00", "LattePort": 6100, "LatteDuration": "00:01:00", "NTttcpPort": 5500, @@ -225,14 +225,14 @@ { "Type": "NetworkingWorkloadExecutor", "Parameters": { - "Scenario": "CPS_T16", - "ToolName": "CPS", - "PackageName": "Networking", - "Connections": 16, - "TestDuration": "$.Parameters.CpsDuration", + "Scenario": "NCPS_T16", + "ToolName": "NCPS", + "PackageName": "networking", + "ThreadCount": 16, + "TestDuration": "$.Parameters.NcpsDuration", "WarmupTime": "00:01:00", - "Port": "$.Parameters.CpsPort", - "ProfilingScenario": "CPS_T16", + "Port": "$.Parameters.NcpsPort", + "ProfilingScenario": "NCPS_T16", "ProfilingEnabled": "$.Parameters.ProfilingEnabled", "ProfilingMode": "$.Parameters.ProfilingMode" } @@ -364,4 +364,4 @@ } } ] -} \ No newline at end of file +} diff --git a/website/docs/developing/0050-develop-profiling-monitor.md b/website/docs/developing/0050-develop-profiling-monitor.md index 24333ff6f6..fab8312482 100644 --- a/website/docs/developing/0050-develop-profiling-monitor.md +++ b/website/docs/developing/0050-develop-profiling-monitor.md @@ -270,10 +270,10 @@ VirtualClient.exe --profile=PERF-NETWORK.json --profile=MONITORS-PROFILING.json { "Type": "NetworkingWorkloadExecutor", "Parameters": { - "Scenario": "CPS", - "ToolName": "CPS", - "PackageName": "Networking", - "Connections": "16", + "Scenario": "NCPS_T16", + "ToolName": "NCPS", + "PackageName": "networking", + "ThreadCount": 16, "TestDuration": "300", "WarmupTime" : "60", "ProfilingEnabled": "$.Parameters.ProfilingEnabled", @@ -354,4 +354,4 @@ shows how to do this: } ``` - \ No newline at end of file + diff --git a/website/docs/guides/0200-usage-examples.md b/website/docs/guides/0200-usage-examples.md index 242245541f..19ce62b5a6 100644 --- a/website/docs/guides/0200-usage-examples.md +++ b/website/docs/guides/0200-usage-examples.md @@ -22,7 +22,7 @@ VirtualClient.exe --profile=PERF-CPU-COREMARK.json --timeout=180 ``` ## Scenario: Running a Client Server Workload -Some workload profiles require multiple systems to operate. For example the CPS, NTttcp and SockPerf workloads requires a client system and a server +Some workload profiles require multiple systems to operate. For example the NCPS, NTttcp and SockPerf workloads require a client system and a server system to be valid. Multi-system workload profiles require and environment layout to be supplied to the Virtual Client. An environment layout describes the topology...where the other Virtual Client instances are and what roles they will play. To get familiar with defining an environment layouts, see the documentation below. Note that each of the workload profiles has documentation that @@ -302,4 +302,4 @@ can be used to define alternate locations for dependencies: # Note that the temp directory can be defined on the command line as well. Temp directories # defined on the command line take priority over those defined by the environment variable. /home/user/virtualclient$ ./VirtualClient --profile=PERF-CPU-OPENSSL.json --timeout=120 --temp-dir="/home/user/temp" - ``` \ No newline at end of file + ``` diff --git a/website/docs/overview/overview.md b/website/docs/overview/overview.md index 3f46d2d488..a17c01fd78 100644 --- a/website/docs/overview/overview.md +++ b/website/docs/overview/overview.md @@ -36,7 +36,7 @@ for using them. | [BlenderBenchmark](https://microsoft.github.io/VirtualClient/docs/workloads/blenderbenchmark) | GPU/Graphics Rendering Performance | win-x64 | [GNU LGPL](https://projects.blender.org/infrastructure/blender-open-data/src/branch/main/LICENSE) | | [CoreMark](https://microsoft.github.io/VirtualClient/docs/workloads/coremark/coremark) | CPU Performance | linux-x64, linux-arm64 | [Apache+Custom](https://github.com/eembc/coremark/blob/main/LICENSE.md) | | [CoreMark Pro](https://microsoft.github.io/VirtualClient/docs/workloads/coremark) | Precision CPU | linux-x64, linux-arm64, win-x64, win-arm64 | [Apache+Custom](https://github.com/eembc/coremark-pro/blob/main/LICENSE.md) | -| [CPS](https://microsoft.github.io/VirtualClient/docs/workloads/network-suite) | Network Connection Reliability | linux-x64, linux-arm64, win-x64, win-arm64 | Microsoft-Developed | +| [NCPS](https://microsoft.github.io/VirtualClient/docs/workloads/network-suite) | Network Connection Reliability | linux-x64, linux-arm64, win-x64, win-arm64 | Microsoft-Developed | | [DCGMI](https://microsoft.github.io/VirtualClient/docs/workloads/dcgmi)| GPU Qualification| linux-x64 | [Apache-2.0](https://github.com/NVIDIA/DCGM/blob/master/LICENSE) | [DeathStarBench](https://microsoft.github.io/VirtualClient/docs/workloads/deathstarbench) | Docker Swarm/Container Microservices | linux-x64, linux-arm64, win-x64, win-arm64 | [Apache-2.0](https://github.com/delimitrou/DeathStarBench/blob/master/LICENSE) | | [DiskSpd](https://microsoft.github.io/VirtualClient/docs/workloads/diskspd) | Disk I/O Performance | win-x64, win-arm64 | [MIT](https://github.com/microsoft/diskspd/blob/master/LICENSE) | @@ -100,4 +100,4 @@ Your use of the software operates as your consent to these practices. This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft trademarks or logos is subject to and must follow [Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general). Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. -Any use of third-party trademarks or logos are subject to those third-party's policies. \ No newline at end of file +Any use of third-party trademarks or logos are subject to those third-party's policies. diff --git a/website/docs/workloads/network-suite/network-suite-profiles.md b/website/docs/workloads/network-suite/network-suite-profiles.md index 5b176787fe..3390ffd068 100644 --- a/website/docs/workloads/network-suite/network-suite-profiles.md +++ b/website/docs/workloads/network-suite/network-suite-profiles.md @@ -43,8 +43,8 @@ VirtualClient.exe --profile=PERF-NETWORK.json --system=Demo --timeout=1440 --cli ``` ## PERF-NETWORK.json -Runs the suite of workloads on the system to evaluate the average and peak performance of the network hardware and stack. On Linux systems, the CPS, NTttcp and SockPerf workloads -are used. On Windows systems the CPS, NTttcp and Latte workloads are used. +Runs the suite of workloads on the system to evaluate the average and peak performance of the network hardware and stack. On Linux systems, the NCPS, NTttcp and SockPerf workloads +are used. On Windows systems the NCPS, NTttcp and Latte workloads are used. :::danger *By default, the systems have settings applied to ensure the OS is configured for full network performance. These settings require the systems to be rebooted in order to be applied. @@ -99,7 +99,6 @@ sysctl -w net.core.busy_read=50 * **Scenarios** The following scenarios are covered by this workload profile. - * CPS. Connection establishment speed and reliability, 16 concurrent connections. * NCPS. Connection establishment speed and reliability, 16 concurrent connections. * NTttcp. Network communications throughput and bandwidth with the following scenarios. * TCP protocol, 4K network buffer, 1 thread @@ -131,8 +130,6 @@ sysctl -w net.core.busy_read=50 | ConfigureNetwork | Optional. True to configure the system network settings for peak performance (will reboot system). False to make no changes to the system network settings. | True (will reboot system) | | EnableBusyPoll | Optional. True to configure busy poll in the system network settings for Linux systems. False to make no changes. This setting depends upon 'ConfigureNetwork' = true. | True | | DisableFirewall | Optional. True to disable the firewall on Linux systems. False to make no changes. This setting depends upon 'ConfigureNetwork' = true. | True | - | CpsDuration | Optional. The amount of time (in seconds) to run the CPS workload. | 300 secs | - | CpsPort | Optional. The starting port on which connections will be established between client and server when running the CPS workload. The CPS workload will use connections on additional ports starting with this port for each of the number of connections (e.g. 16) defined in the component/action. | 7201 | | NcpsDuration | Optional. The amount of time (in seconds) to run the NCPS workload. | 300 secs | | NcpsPort | Optional. The starting port on which connections will be established between client and server when running the NCPS workload. The NCPS workload will use connections on additional ports starting with this port for each of the number of connections (e.g. 16) defined in the component/action. | 9800 | | LattePort | Optional. The starting port on which connections will be established between client and server when running the Latte workload. The Latte workload will use connections on additional ports starting with this port. | 6100 | @@ -165,4 +162,4 @@ sysctl -w net.core.busy_read=50 # On the Server role system ./VirtualClient --profile=PERF-NETWORK.json --system=Demo --timeout=1440 --client-id=Server01 --layout="/any/path/to/layout.json" - ``` \ No newline at end of file + ``` diff --git a/website/docs/workloads/network-suite/network-suite.md b/website/docs/workloads/network-suite/network-suite.md index a53b2b9598..d1fe9d1504 100644 --- a/website/docs/workloads/network-suite/network-suite.md +++ b/website/docs/workloads/network-suite/network-suite.md @@ -5,7 +5,7 @@ and reliability. The workloads that are a part of the suite include: * **CPS** - This workload that is used to measure network socket connection establishment efficiencies and reliability between a client and a server. + This legacy workload measures network socket connection establishment efficiencies and reliability between a client and a server. It is deprecated in favor of NCPS for the standard PERF-NETWORK profile. * **NCPS** This workload (New Connections Per Second) is an enhanced version of CPS that provides more granular connection establishment metrics, throughput measurements, and advanced statistical analysis between a client and a server. @@ -22,9 +22,6 @@ The workloads that are a part of the suite include: ## What is Being Tested? The following performance analysis scenarios are covered as part of the network workload suite. -* CPS - * Network connection establishment reliability and speed on both Unix/Linux and Windows systems. - * NCPS * Enhanced network connection establishment performance with detailed metrics on both Unix/Linux and Windows systems. * Connection throughput with RX/TX bandwidth measurements in Gbps. @@ -215,4 +212,4 @@ For the Latte and SockPerf workloads, the client measurements are the only ones | SockPerf Client | Latency-P99.99 | 4239.64 | 8768.027 | 6637.732552238807 | | | SockPerf Client | Latency-P99.999 | 9222.604 | 23002.865 | 15495.719104477614 | | | SockPerf Client | Latency-Stdev | 116.05 | 184.24 | 158.41534328358214 | | -| SockPerf Server | N/A. No server-side metrics. | | | | | \ No newline at end of file +| SockPerf Server | N/A. No server-side metrics. | | | | |