Skip to content

Commit

Permalink
Apply a few standardizations to DiskSpd and FIO profiles to reduce te…
Browse files Browse the repository at this point in the history 
…chnical/cleanup debt.
  • Loading branch information
@brdeyo
brdeyo committed Jul 16, 2024
1 parent 3e9f670 commit 78e96db
Show file tree
Hide file tree
Showing 11 changed files with 414 additions and 546 deletions.
1 change: 1 addition & 0 deletions src/VirtualClient/VirtualClient.Actions.FunctionalTests/DiskSpdProfileTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,7 @@ private static IEnumerable<string> GetDiskSpdStressProfileExpectedCommands()

// After the disk fill, we execute the DiskSpd commands.
// Random Write tests
// e.g. -c496G -b4K -r4K -t4 -o128 -w100 -d300 -Suw -W15 -D -L -Rtext D:\\diskspd-test.dat E:\\diskspd-test.dat F:\\diskspd-test.dat
$"-c496G -b4K -r4K -t[0-9]+ -o[0-9]+ -w100 -d300 -Suw -W15 -D -L -Rtext",
$"-c496G -b8K -r4K -t[0-9]+ -o[0-9]+ -w100 -d300 -Suw -W15 -D -L -Rtext",
$"-c496G -b12K -r4K -t[0-9]+ -o[0-9]+ -w100 -d300 -Suw -W15 -D -L -Rtext",
Expand Down
32 changes: 16 additions & 16 deletions src/VirtualClient/VirtualClient.Actions.FunctionalTests/SysbenchProfileTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ public void SetupFixture()
}

[Test]
[TestCase("PERF-MYSQL-OLTP-SYSBENCH.json")]
[TestCase("PERF-MYSQL-SYSBENCH-OLTP.json")]
public void MySQLSysbenchOLTPWorkloadProfileParametersAreInlinedCorrectly(string profile)
{
using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.fixture.Dependencies))
Expand All @@ -46,7 +46,7 @@ public void MySQLSysbenchOLTPWorkloadProfileParametersAreInlinedCorrectly(string
}

[Test]
[TestCase("PERF-MYSQL-OLTP-SYSBENCH.json", PlatformID.Unix, Architecture.X64)]
[TestCase("PERF-MYSQL-SYSBENCH-OLTP.json", PlatformID.Unix, Architecture.X64)]
public void MySQLSysbenchOLTPWorkloadProfileActionsWillNotBeExecutedIfTheWorkloadPackageDoesNotExist(string profile, PlatformID platform, Architecture architecture)
{
this.fixture.Setup(platform);
Expand All @@ -62,7 +62,7 @@ public void MySQLSysbenchOLTPWorkloadProfileActionsWillNotBeExecutedIfTheWorkloa
}

[Test]
[TestCase("PERF-MYSQL-OLTP-SYSBENCH.json", PlatformID.Unix, Architecture.X64)]
[TestCase("PERF-MYSQL-SYSBENCH-OLTP.json", PlatformID.Unix, Architecture.X64)]
public async Task MySQLSysbenchOLTPWorkloadProfileExecutesTheExpectedWorkloadsOnUnixPlatform(string profile, PlatformID platform, Architecture architecture)
{
this.fixture.Setup(platform, architecture, this.clientAgentId).SetupLayout(
Expand Down Expand Up @@ -96,7 +96,7 @@ public async Task MySQLSysbenchOLTPWorkloadProfileExecutesTheExpectedWorkloadsOn
}

[Test]
[TestCase("PERF-MYSQL-OLTP-SYSBENCH.json", PlatformID.Unix, Architecture.X64)]
[TestCase("PERF-MYSQL-SYSBENCH-OLTP.json", PlatformID.Unix, Architecture.X64)]
public async Task MySQLSysbenchOLTPWorkloadProfileExecutesTheExpectedWorkloadsOnSingleVMUnixPlatform(string profile, PlatformID platform, Architecture architecture)
{
this.fixture.Setup(platform, architecture, this.clientAgentId).SetupLayout(
Expand Down Expand Up @@ -142,7 +142,7 @@ public async Task MySQLSysbenchOLTPWorkloadProfileExecutesTheExpectedWorkloadsOn
}

[Test]
[TestCase("PERF-MYSQL-TPCC-SYSBENCH.json")]
[TestCase("PERF-MYSQL-SYSBENCH-TPCC.json")]
public void MySQLSysbenchTPCCWorkloadProfileParametersAreInlinedCorrectly(string profile)
{
using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.fixture.Dependencies))
Expand All @@ -152,7 +152,7 @@ public void MySQLSysbenchTPCCWorkloadProfileParametersAreInlinedCorrectly(string
}

[Test]
[TestCase("PERF-MYSQL-TPCC-SYSBENCH.json", PlatformID.Unix, Architecture.X64)]
[TestCase("PERF-MYSQL-SYSBENCH-TPCC.json", PlatformID.Unix, Architecture.X64)]
public void MySQLSysbenchTPCCWorkloadProfileActionsWillNotBeExecutedIfTheWorkloadPackageDoesNotExist(string profile, PlatformID platform, Architecture architecture)
{
this.fixture.Setup(platform);
Expand All @@ -168,7 +168,7 @@ public void MySQLSysbenchTPCCWorkloadProfileActionsWillNotBeExecutedIfTheWorkloa
}

[Test]
[TestCase("PERF-MYSQL-TPCC-SYSBENCH.json", PlatformID.Unix, Architecture.X64)]
[TestCase("PERF-MYSQL-SYSBENCH-TPCC.json", PlatformID.Unix, Architecture.X64)]
public async Task MySQLSysbenchTPCCWorkloadProfileExecutesTheExpectedWorkloadsOnUnixPlatform(string profile, PlatformID platform, Architecture architecture)
{
this.fixture.Setup(platform, architecture, this.clientAgentId).SetupLayout(
Expand Down Expand Up @@ -202,7 +202,7 @@ public async Task MySQLSysbenchTPCCWorkloadProfileExecutesTheExpectedWorkloadsOn
}

[Test]
[TestCase("PERF-MYSQL-TPCC-SYSBENCH.json", PlatformID.Unix, Architecture.X64)]
[TestCase("PERF-MYSQL-SYSBENCH-TPCC.json", PlatformID.Unix, Architecture.X64)]
public async Task MySQLSysbenchTPCCWorkloadProfileExecutesTheExpectedWorkloadsOnMySQLSingleVMUnixPlatform(string profile, PlatformID platform, Architecture architecture)
{
this.fixture.Setup(platform, architecture, this.clientAgentId).SetupLayout(
Expand Down Expand Up @@ -248,7 +248,7 @@ public async Task MySQLSysbenchTPCCWorkloadProfileExecutesTheExpectedWorkloadsOn
}

[Test]
[TestCase("PERF-POSTGRESQL-OLTP-SYSBENCH.json")]
[TestCase("PERF-POSTGRESQL-SYSBENCH-OLTP.json")]
public void PostgreSQLSysbenchOLTPWorkloadProfileParametersAreInlinedCorrectly(string profile)
{
using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.fixture.Dependencies))
Expand All @@ -258,7 +258,7 @@ public void PostgreSQLSysbenchOLTPWorkloadProfileParametersAreInlinedCorrectly(s
}

[Test]
[TestCase("PERF-POSTGRESQL-OLTP-SYSBENCH.json", PlatformID.Unix, Architecture.X64)]
[TestCase("PERF-POSTGRESQL-SYSBENCH-OLTP.json", PlatformID.Unix, Architecture.X64)]
public void PostgreSQLSysbenchOLTPWorkloadProfileActionsWillNotBeExecutedIfTheWorkloadPackageDoesNotExist(string profile, PlatformID platform, Architecture architecture)
{
this.fixture.Setup(platform);
Expand All @@ -274,7 +274,7 @@ public void PostgreSQLSysbenchOLTPWorkloadProfileActionsWillNotBeExecutedIfTheWo
}

[Test]
[TestCase("PERF-POSTGRESQL-OLTP-SYSBENCH.json", PlatformID.Unix, Architecture.X64)]
[TestCase("PERF-POSTGRESQL-SYSBENCH-OLTP.json", PlatformID.Unix, Architecture.X64)]
public async Task PostgreSQLSysbenchOLTPWorkloadProfileExecutesTheExpectedWorkloadsOnUnixPlatform(string profile, PlatformID platform, Architecture architecture)
{
this.fixture.Setup(platform, architecture, this.clientAgentId).SetupLayout(
Expand Down Expand Up @@ -308,7 +308,7 @@ public async Task PostgreSQLSysbenchOLTPWorkloadProfileExecutesTheExpectedWorklo
}

[Test]
[TestCase("PERF-POSTGRESQL-OLTP-SYSBENCH.json", PlatformID.Unix, Architecture.X64)]
[TestCase("PERF-POSTGRESQL-SYSBENCH-OLTP.json", PlatformID.Unix, Architecture.X64)]
public async Task PostgreSQLSysbenchOLTPWorkloadProfileExecutesTheExpectedWorkloadsOnSingleVMUnixPlatform(string profile, PlatformID platform, Architecture architecture)
{
this.fixture.Setup(platform, architecture, this.clientAgentId).SetupLayout(
Expand Down Expand Up @@ -354,7 +354,7 @@ public async Task PostgreSQLSysbenchOLTPWorkloadProfileExecutesTheExpectedWorklo
}

[Test]
[TestCase("PERF-POSTGRESQL-TPCC-SYSBENCH.json")]
[TestCase("PERF-POSTGRESQL-SYSBENCH-TPCC.json")]
public void PostgreSQLSysbenchTPCCWorkloadProfileParametersAreInlinedCorrectly(string profile)
{
using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.fixture.Dependencies))
Expand All @@ -364,7 +364,7 @@ public void PostgreSQLSysbenchTPCCWorkloadProfileParametersAreInlinedCorrectly(s
}

[Test]
[TestCase("PERF-POSTGRESQL-TPCC-SYSBENCH.json", PlatformID.Unix, Architecture.X64)]
[TestCase("PERF-POSTGRESQL-SYSBENCH-TPCC.json", PlatformID.Unix, Architecture.X64)]
public void PostgreSQLSysbenchTPCCWorkloadProfileActionsWillNotBeExecutedIfTheWorkloadPackageDoesNotExist(string profile, PlatformID platform, Architecture architecture)
{
this.fixture.Setup(platform);
Expand All @@ -380,7 +380,7 @@ public void PostgreSQLSysbenchTPCCWorkloadProfileActionsWillNotBeExecutedIfTheWo
}

[Test]
[TestCase("PERF-POSTGRESQL-TPCC-SYSBENCH.json", PlatformID.Unix, Architecture.X64)]
[TestCase("PERF-POSTGRESQL-SYSBENCH-TPCC.json", PlatformID.Unix, Architecture.X64)]
public async Task PostgreSQLSysbenchTPCCWorkloadProfileExecutesTheExpectedWorkloadsOnUnixPlatform(string profile, PlatformID platform, Architecture architecture)
{
this.fixture.Setup(platform, architecture, this.clientAgentId).SetupLayout(
Expand Down Expand Up @@ -414,7 +414,7 @@ public async Task PostgreSQLSysbenchTPCCWorkloadProfileExecutesTheExpectedWorklo
}

[Test]
[TestCase("PERF-POSTGRESQL-TPCC-SYSBENCH.json", PlatformID.Unix, Architecture.X64)]
[TestCase("PERF-POSTGRESQL-SYSBENCH-TPCC.json", PlatformID.Unix, Architecture.X64)]
public async Task PostgreSQLSysbenchTPCCWorkloadProfileExecutesTheExpectedWorkloadsOnSingleVMUnixPlatform(string profile, PlatformID platform, Architecture architecture)
{
this.fixture.Setup(platform, architecture, this.clientAgentId).SetupLayout(
Expand Down
8 changes: 4 additions & 4 deletions ...Client/VirtualClient.Actions.FunctionalTests/VirtualClient.Actions.FunctionalTests.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,10 +202,10 @@
<Content Include="..\VirtualClient.Main\profiles\PERF-MEMCACHED.json" Link="Profiles\PERF-MEMCACHED.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="..\VirtualClient.Main\profiles\PERF-MYSQL-OLTP-SYSBENCH.json" Link="Profiles\PERF-MYSQL-OLTP-SYSBENCH.json">
<Content Include="..\VirtualClient.Main\profiles\PERF-MYSQL-SYSBENCH-OLTP.json" Link="Profiles\PERF-MYSQL-SYSBENCH-OLTP.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="..\VirtualClient.Main\profiles\PERF-MYSQL-TPCC-SYSBENCH.json" Link="Profiles\PERF-MYSQL-TPCC-SYSBENCH.json">
<Content Include="..\VirtualClient.Main\profiles\PERF-MYSQL-SYSBENCH-TPCC.json" Link="Profiles\PERF-MYSQL-SYSBENCH-TPCC.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="..\VirtualClient.Main\profiles\PERF-NETWORK-DEATHSTARBENCH.json" Link="Profiles\PERF-NETWORK-DEATHSTARBENCH.json">
Expand All @@ -223,10 +223,10 @@
<Content Include="..\VirtualClient.Main\profiles\PERF-POSTGRESQL-HAMMERDB-TPCC.json" Link="Profiles\PERF-POSTGRESQL-HAMMERDB-TPCC.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="..\VirtualClient.Main\profiles\PERF-POSTGRESQL-OLTP-SYSBENCH.json" Link="Profiles\PERF-POSTGRESQL-OLTP-SYSBENCH.json">
<Content Include="..\VirtualClient.Main\profiles\PERF-POSTGRESQL-SYSBENCH-OLTP.json" Link="Profiles\PERF-POSTGRESQL-SYSBENCH-OLTP.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="..\VirtualClient.Main\profiles\PERF-POSTGRESQL-TPCC-SYSBENCH.json" Link="Profiles\PERF-POSTGRESQL-TPCC-SYSBENCH.json">
<Content Include="..\VirtualClient.Main\profiles\PERF-POSTGRESQL-SYSBENCH-TPCC.json" Link="Profiles\PERF-POSTGRESQL-SYSBENCH-TPCC.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="..\VirtualClient.Main\profiles\PERF-SPECCPU-FPRATE.json" Link="Profiles\PERF-SPECCPU-FPRATE.json">
Expand Down
53 changes: 7 additions & 46 deletions src/VirtualClient/VirtualClient.Actions.UnitTests/DiskSpd/DiskSpdExecutorTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ public void SetupDefaultBehavior()
{ nameof(DiskSpdExecutor.PackageName), "diskspd" },
{ nameof(DiskSpdExecutor.CommandLine), "-c4G -b4K -r4K -t1 -o1 -w100" },
{ nameof(DiskSpdExecutor.ProcessModel), WorkloadProcessModel.SingleProcess },
{ nameof(DiskSpdExecutor.TestName), "diskspd_randwrite_4GB_direct" }
{ nameof(DiskSpdExecutor.MetricScenario), "diskspd_randwrite_4GB_direct" }
};

string workingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
Expand All @@ -63,7 +63,7 @@ public void SetupDefaultBehavior()
public void DiskSpdExecutorAppliesConfigurationParametersCorrectly()
{
this.profileParameters[nameof(DiskSpdExecutor.CommandLine)] = "-c{FileSize} -b8K -r4K -t{ThreadCount} -o{QueueDepth} -w0 -d480 -Suw -W15 -D -L -Rtext";
this.profileParameters[nameof(DiskSpdExecutor.TestName)] = "diskspd_randread_{FileSize}_8k_d{QueueDepth}_th{ThreadCount}";
this.profileParameters[nameof(DiskSpdExecutor.MetricScenario)] = "diskspd_randread_{FileSize}_8k_d{QueueDepth}_th{ThreadCount}";
this.profileParameters[nameof(DiskSpdExecutor.FileSize)] = "496G";
this.profileParameters[nameof(DiskSpdExecutor.QueueDepth)] = 16;
this.profileParameters[nameof(DiskSpdExecutor.ThreadCount)] = 32;
Expand All @@ -73,7 +73,7 @@ public void DiskSpdExecutorAppliesConfigurationParametersCorrectly()
executor.EvaluateParametersAsync(EventContext.None);

string commandLine = executor.Parameters[nameof(DiskSpdExecutor.CommandLine)].ToString();
string testName = executor.Parameters[nameof(DiskSpdExecutor.TestName)].ToString();
string testName = executor.Parameters[nameof(DiskSpdExecutor.MetricScenario)].ToString();

Assert.AreEqual($"-c496G -b8K -r4K -t32 -o16 -w0 -d480 -Suw -W15 -D -L -Rtext", commandLine);
Assert.AreEqual($"diskspd_randread_496G_8k_d16_th32", testName);
Expand All @@ -84,15 +84,15 @@ public void DiskSpdExecutorAppliesConfigurationParametersCorrectly()
public void DiskSpdExecutorAppliesConfigurationParametersCorrectly_DiskFillScenario1()
{
this.profileParameters[nameof(DiskSpdExecutor.CommandLine)] = "-c{DiskFillSize} -b256K -si4K -t1 -o64 -w100 -Suw -W15 -D -L";
this.profileParameters[nameof(DiskSpdExecutor.TestName)] = "disk_fill";
this.profileParameters[nameof(DiskSpdExecutor.MetricScenario)] = "disk_fill";
this.profileParameters[nameof(DiskSpdExecutor.DiskFillSize)] = "496G";

using (TestDiskSpdExecutor executor = new TestDiskSpdExecutor(this.fixture.Dependencies, this.profileParameters))
{
executor.EvaluateParametersAsync(EventContext.None);

string commandLine = executor.Parameters[nameof(DiskSpdExecutor.CommandLine)].ToString();
string testName = executor.Parameters[nameof(DiskSpdExecutor.TestName)].ToString();
string testName = executor.Parameters[nameof(DiskSpdExecutor.MetricScenario)].ToString();

Assert.AreEqual($"-c496G -b256K -si4K -t1 -o64 -w100 -Suw -W15 -D -L", commandLine);
Assert.AreEqual($"disk_fill", testName);
Expand All @@ -108,7 +108,7 @@ public void DiskSpdExecutorCorrectsTheDiskFillSizeAndFileSizeParameters(string s
// Note that this is not a valid DiskSpd command line. This is used ONLY to ensure
// the file sizes are corrected.
this.profileParameters[nameof(DiskSpdExecutor.CommandLine)] = "-c{DiskFillSize} -c{FileSize} -b256K -si4K -t1 -o64 -w100 -Suw -W15 -D -L";
this.profileParameters[nameof(DiskSpdExecutor.TestName)] = "disk_fill";
this.profileParameters[nameof(DiskSpdExecutor.MetricScenario)] = "disk_fill";

// The disk fill size '496GB' is not supported. It should be formatted to '496G'.
this.profileParameters[nameof(DiskSpdExecutor.DiskFillSize)] = size;
Expand All @@ -119,52 +119,13 @@ public void DiskSpdExecutorCorrectsTheDiskFillSizeAndFileSizeParameters(string s
executor.EvaluateParametersAsync(EventContext.None);

string commandLine = executor.Parameters[nameof(DiskSpdExecutor.CommandLine)].ToString();
string testName = executor.Parameters[nameof(DiskSpdExecutor.TestName)].ToString();
string testName = executor.Parameters[nameof(DiskSpdExecutor.MetricScenario)].ToString();

Assert.AreEqual($"-c{correctedSize} -c{correctedSize} -b256K -si4K -t1 -o64 -w100 -Suw -W15 -D -L", commandLine);
Assert.AreEqual($"disk_fill", testName);
}
}

[Test]
public void DiskSpdExecutorAppliesConfigurationParametersCorrectly_StressConfiguration()
{
// Stress Configuration -> Stress Profile
this.profileParameters[nameof(DiskSpdExecutor.CommandLine)] = "-c{FileSize} -b8K -r4K -t{ThreadCount} -o{QueueDepth} -w0 -d480 -Suw -W15 -D -L -Rtext";
this.profileParameters[nameof(DiskSpdExecutor.TestName)] = "diskspd_randread_{FileSize}_8k_d{QueueDepth}_th{ThreadCount}";
this.profileParameters[nameof(DiskSpdExecutor.FileSize)] = "496G";
this.profileParameters[nameof(DiskSpdExecutor.Configuration)] = "Stress";

using (TestDiskSpdExecutor executor = new TestDiskSpdExecutor(this.fixture.Dependencies, this.profileParameters))
{
executor.EvaluateParametersAsync(EventContext.None);

string commandLine = executor.Parameters[nameof(DiskSpdExecutor.CommandLine)].ToString();
string testName = executor.Parameters[nameof(DiskSpdExecutor.TestName)].ToString();

int logicalCores = Environment.ProcessorCount;
int threads = logicalCores / 2;
int queueDepth = 512 / threads;

Assert.AreEqual($"-c496G -b8K -r4K -t{threads} -o{queueDepth} -w0 -d480 -Suw -W15 -D -L -Rtext", commandLine);
Assert.AreEqual($"diskspd_randread_496G_8k_d{queueDepth}_th{threads}", testName);
}
}

[Test]
public void DiskSpdExecutorThrowsIfAnUnsupportedConfigurationIsDefined()
{
this.profileParameters[nameof(DiskSpdExecutor.Configuration)] = "ConfigurationNotSupported";

using (TestDiskSpdExecutor executor = new TestDiskSpdExecutor(this.fixture.Dependencies, this.profileParameters))
{
WorkloadException error = Assert.ThrowsAsync<WorkloadException>(
() => executor.EvaluateParametersAsync(EventContext.None));

Assert.AreEqual(ErrorReason.InvalidProfileDefinition, error.Reason);
}
}

[Test]
public void DiskSpdExecutorValidatesRequiredProfileParameters()
{
Expand Down
Loading

0 comments on commit 78e96db

Please sign in to comment.