Skip to content

Extending support for Superbench on GB200 (linux-arm64) #509

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Extending support for Superbench on GB200 (linux-arm64) #509

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b312939
Initial commit
sasanka-grs-work May 13, 2025
f17532e
Made changes 24 June
sasanka-grs-work Jun 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
292 changes: 262 additions & 30 deletions ...rtualClient/VirtualClient.Actions.UnitTests/SuperBenchmark/SuperBenchmarkExecutorTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@ namespace VirtualClient.Actions
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.CodeAnalysis.CSharp.Syntax;
using Microsoft.Extensions.DependencyInjection;
using Moq;
using Newtonsoft.Json.Linq;
Expand All @@ -26,36 +28,18 @@ public class SuperBenchmarkExecutorTests
private DependencyPath mockPackage;

[SetUp]
public void SetupDefaultBehavior()
public void SetupTests()
{
this.mockFixture = new MockFixture();
this.mockFixture.Setup(PlatformID.Unix);
this.mockPackage = new DependencyPath("SuperBenchmark", this.mockFixture.PlatformSpecifics.GetPackagePath("superbenchmark"));

this.mockFixture.PackageManager.OnGetPackage().ReturnsAsync(this.mockPackage);

this.mockFixture.File.Reset();
this.mockFixture.File.Setup(f => f.Exists(It.IsAny<string>()))
.Returns(true);
this.mockFixture.Directory.Setup(f => f.Exists(It.IsAny<string>()))
.Returns(true);
this.mockFixture.Directory.Setup(f => f.Exists(It.IsRegex("superbenchmark")))
.Returns(false);

this.mockFixture.FileSystem.SetupGet(fs => fs.File).Returns(this.mockFixture.File.Object);

this.mockFixture.Parameters = new Dictionary<string, IConvertible>()
{
{ nameof(SuperBenchmarkExecutor.Version), "0.0.1" },
{ nameof(SuperBenchmarkExecutor.ContainerVersion), "testContainer" },
{ nameof(SuperBenchmarkExecutor.ConfigurationFile), "Test.yaml" },
{ nameof(SuperBenchmarkExecutor.Username), "testuser" }
};
}

[Test]
public void SuperBenchmarkStateIsSerializeable()
[TestCase(Architecture.X64)]
[TestCase(Architecture.Arm64)]
public void SuperBenchmarkStateIsSerializeable(Architecture architecture)
{
SetupDefaultMockBehavior(architecture);

State state = new State(new Dictionary<string, IConvertible>
{
["SuperBenchmarkInitialized"] = true
Expand All @@ -69,8 +53,10 @@ public void SuperBenchmarkStateIsSerializeable()
}

[Test]
public async Task SuperBenchmarkExecutorClonesTheExpectedRepoContents()
public async Task SuperBenchmarkExecutorClonesTheExpectedRepoContentsOnX64Architecture()
{
SetupDefaultMockBehavior(Architecture.X64);

ProcessStartInfo expectedInfo = new ProcessStartInfo();
this.mockFixture.Parameters = new Dictionary<string, IConvertible>()
{
Expand Down Expand Up @@ -111,8 +97,54 @@ public async Task SuperBenchmarkExecutorClonesTheExpectedRepoContents()
}

[Test]
public async Task SuperBenchmarkExecutorUsesTheExpectedScriptFilesOnExecution()
public async Task SuperBenchmarkExecutorPullsTheExpectedDockerImageContentsOnArm64Architecture()
{
SetupDefaultMockBehavior(Architecture.Arm64);

ProcessStartInfo expectedInfo = new ProcessStartInfo();
this.mockFixture.Parameters = new Dictionary<string, IConvertible>()
{
{ nameof(SuperBenchmarkExecutor.Version), "1.2.3" },
{ nameof(SuperBenchmarkExecutor.ContainerVersion), "testContainer" },
{ nameof(SuperBenchmarkExecutor.ConfigurationFile), "Test.yaml" },
{ nameof(SuperBenchmarkExecutor.Username), "testuser" }
};
string expectedCommand = $"sudo docker pull {this.mockFixture.Parameters[nameof(SuperBenchmarkExecutor.ContainerVersion)]}";

bool commandExecuted = false;
this.mockFixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) =>
{
if (expectedCommand == $"{exe} {arguments}")
{
commandExecuted = true;
}

return new InMemoryProcess
{
StartInfo = new ProcessStartInfo
{
FileName = exe,
Arguments = arguments
},
ExitCode = 0,
OnStart = () => true,
OnHasExited = () => true
};
};

using (TestSuperBenchmarkExecutor superBenchmarkExecutor = new TestSuperBenchmarkExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters))
{
await superBenchmarkExecutor.ExecuteAsync(CancellationToken.None).ConfigureAwait(false);
}

Assert.IsTrue(commandExecuted);
}

[Test]
public async Task SuperBenchmarkExecutorUsesTheExpectedScriptFilesOnExecutionOnX64Architecture()
{
SetupDefaultMockBehavior(Architecture.X64);

ProcessStartInfo expectedInfo = new ProcessStartInfo();
string expectedCommand = $"sudo bash initialize.sh testuser";

Expand Down Expand Up @@ -146,8 +178,10 @@ public async Task SuperBenchmarkExecutorUsesTheExpectedScriptFilesOnExecution()
}

[Test]
public async Task SuperBenchmarkExecutorDeploySuperBenchContainer()
public async Task SuperBenchmarkExecutorDeploySuperBenchContainerOnX64Architecture()
{
SetupDefaultMockBehavior(Architecture.X64);

ProcessStartInfo expectedInfo = new ProcessStartInfo();
string expectedCommand = $"sb deploy --host-list localhost -i testContainer";

Expand Down Expand Up @@ -181,8 +215,48 @@ public async Task SuperBenchmarkExecutorDeploySuperBenchContainer()
}

[Test]
public async Task SuperBenchmarkExecutorRunsTheExpectedWorkloadCommand()
public async Task SuperBenchmarkExecutorRunsDockerContainerInDetachedModeForSetupOnArm64Architecture()
{
SetupDefaultMockBehavior(Architecture.Arm64);

ProcessStartInfo expectedInfo = new ProcessStartInfo();
string expectedPath = this.mockFixture.PlatformSpecifics.Combine(this.mockFixture.PlatformSpecifics.PackagesDirectory, "superbenchmark");
string expectedCommand = $"sudo docker run -itd --name=sb-dev --privileged --net=host --ipc=host --gpus=all -w /root -v {expectedPath}:/mnt testContainer";

bool commandExecuted = false;
this.mockFixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) =>
{
if (expectedCommand == $"{exe} {arguments}")
{
commandExecuted = true;
}

return new InMemoryProcess
{
StartInfo = new ProcessStartInfo
{
FileName = exe,
Arguments = arguments
},
ExitCode = 0,
OnStart = () => true,
OnHasExited = () => true
};
};

using (TestSuperBenchmarkExecutor superBenchmarkExecutor = new TestSuperBenchmarkExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters))
{
await superBenchmarkExecutor.ExecuteAsync(CancellationToken.None).ConfigureAwait(false);
}

Assert.IsTrue(commandExecuted);
}

[Test]
public async Task SuperBenchmarkExecutorRunsTheExpectedWorkloadCommandOnX64Architecture()
{
SetupDefaultMockBehavior(Architecture.X64);

ProcessStartInfo expectedInfo = new ProcessStartInfo();
string expectedCommand = $"sb run --host-list localhost -c Test.yaml";

Expand Down Expand Up @@ -216,8 +290,47 @@ public async Task SuperBenchmarkExecutorRunsTheExpectedWorkloadCommand()
}

[Test]
public async Task SuperBenchmarkExecutorExecutesTheCorrectCommandsWithInstallation()
public async Task SuperBenchmarkExecutorRunsTheExpectedWorkloadCommandOnArm64Architecture()
{
SetupDefaultMockBehavior(Architecture.Arm64);

ProcessStartInfo expectedInfo = new ProcessStartInfo();
string expectedCommand = $"sudo docker exec sb-dev sb run --no-docker -l localhost -c /mnt/Test.yaml --output-dir outputs/";

bool commandExecuted = false;
this.mockFixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) =>
{
if (expectedCommand == $"{exe} {arguments}")
{
commandExecuted = true;
}

return new InMemoryProcess
{
StartInfo = new ProcessStartInfo
{
FileName = exe,
Arguments = arguments
},
ExitCode = 0,
OnStart = () => true,
OnHasExited = () => true
};
};

using (TestSuperBenchmarkExecutor superBenchmarkExecutor = new TestSuperBenchmarkExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters))
{
await superBenchmarkExecutor.ExecuteAsync(CancellationToken.None).ConfigureAwait(false);
}

Assert.IsTrue(commandExecuted);
}

[Test]
public async Task SuperBenchmarkExecutorExecutesTheCorrectCommandsWithInstallationOnX64Architecture()
{
SetupDefaultMockBehavior(Architecture.X64);

ProcessStartInfo expectedInfo = new ProcessStartInfo();
List<string> expectedCommands = new List<string>
{
Expand Down Expand Up @@ -261,8 +374,57 @@ public async Task SuperBenchmarkExecutorExecutesTheCorrectCommandsWithInstallati
}

[Test]
public async Task SuperBenchmarkExecutorSkipsInitializationOfTheWorkloadForExecutionAfterTheFirstRun()
public async Task SuperBenchmarkExecutorExecutesTheCorrectCommandsWithInstallationOnArm64Architecture()
{
SetupDefaultMockBehavior(Architecture.Arm64);

ProcessStartInfo expectedInfo = new ProcessStartInfo();
string expectedPath = this.mockFixture.PlatformSpecifics.Combine(this.mockFixture.PlatformSpecifics.PackagesDirectory, "superbenchmark");
List<string> expectedCommands = new List<string>
{
$"sudo chmod -R 2777 \"{this.mockFixture.PlatformSpecifics.CurrentDirectory}\"",
$"sudo docker pull testContainer",
$"sudo docker run -itd --name=sb-dev --privileged --net=host --ipc=host --gpus=all -w /root -v {expectedPath}:/mnt testContainer",
$"sudo docker exec sb-dev sb run --no-docker -l localhost -c /mnt/Test.yaml --output-dir outputs/"
};

int processCount = 0;
this.mockFixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) =>
{
Assert.AreEqual(expectedCommands.ElementAt(processCount), $"{exe} {arguments}");
processCount++;

return new InMemoryProcess
{
StartInfo = new ProcessStartInfo
{
FileName = exe,
Arguments = arguments
},
ExitCode = 0,
OnStart = () => true,
OnHasExited = () => true
};
};

this.mockFixture.StateManager.OnGetState().ReturnsAsync(JObject.FromObject(new SuperBenchmarkExecutor.SuperBenchmarkState()
{
SuperBenchmarkInitialized = false
}));

using (TestSuperBenchmarkExecutor superBenchmarkExecutor = new TestSuperBenchmarkExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters))
{
await superBenchmarkExecutor.ExecuteAsync(CancellationToken.None).ConfigureAwait(false);
}

Assert.IsTrue(processCount == 4);
}

[Test]
public async Task SuperBenchmarkExecutorSkipsInitializationOfTheWorkloadForExecutionAfterTheFirstRunOnX64Architecture()
{
this.SetupDefaultMockBehavior(Architecture.X64);

ProcessStartInfo expectedInfo = new ProcessStartInfo();
List<string> expectedCommands = new List<string>
{
Expand Down Expand Up @@ -301,6 +463,76 @@ public async Task SuperBenchmarkExecutorSkipsInitializationOfTheWorkloadForExecu
Assert.IsTrue(processCount == 1);
}

[Test]
public async Task SuperBenchmarkExecutorSkipsInitializationOfTheWorkloadForExecutionAfterTheFirstRunOnArm64Architecture()
{
SetupDefaultMockBehavior(Architecture.Arm64);

ProcessStartInfo expectedInfo = new ProcessStartInfo();
List<string> expectedCommands = new List<string>
{
$"sb run --host-list localhost -c Test.yaml"
};

int processCount = 0;
this.mockFixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) =>
{
Assert.AreEqual(expectedCommands.ElementAt(processCount), $"{exe} {arguments}");
processCount++;

return new InMemoryProcess
{
StartInfo = new ProcessStartInfo
{
FileName = exe,
Arguments = arguments
},
ExitCode = 0,
OnStart = () => true,
OnHasExited = () => true
};
};

this.mockFixture.StateManager.OnGetState().ReturnsAsync(JObject.FromObject(new SuperBenchmarkExecutor.SuperBenchmarkState()
{
SuperBenchmarkInitialized = true
}));

using (TestSuperBenchmarkExecutor superBenchmarkExecutor = new TestSuperBenchmarkExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters))
{
await superBenchmarkExecutor.ExecuteAsync(CancellationToken.None).ConfigureAwait(false);
}

Assert.IsTrue(processCount == 1);
}

public void SetupDefaultMockBehavior(Architecture architecture)
{
this.mockFixture = new MockFixture();
this.mockFixture.Setup(PlatformID.Unix, architecture);
this.mockPackage = new DependencyPath("SuperBenchmark", this.mockFixture.PlatformSpecifics.GetPackagePath("superbenchmark"));

this.mockFixture.PackageManager.OnGetPackage().ReturnsAsync(this.mockPackage);

this.mockFixture.File.Reset();
this.mockFixture.File.Setup(f => f.Exists(It.IsAny<string>()))
.Returns(true);
this.mockFixture.Directory.Setup(f => f.Exists(It.IsAny<string>()))
.Returns(true);
this.mockFixture.Directory.Setup(f => f.Exists(It.IsRegex("superbenchmark")))
.Returns(false);

this.mockFixture.FileSystem.SetupGet(fs => fs.File).Returns(this.mockFixture.File.Object);

this.mockFixture.Parameters = new Dictionary<string, IConvertible>()
{
{ nameof(SuperBenchmarkExecutor.Version), "0.0.1" },
{ nameof(SuperBenchmarkExecutor.ContainerVersion), "testContainer" },
{ nameof(SuperBenchmarkExecutor.ConfigurationFile), "Test.yaml" },
{ nameof(SuperBenchmarkExecutor.Username), "testuser" }
};
}

private class TestSuperBenchmarkExecutor : SuperBenchmarkExecutor
{
public TestSuperBenchmarkExecutor(IServiceCollection dependencies, IDictionary<string, IConvertible> parameters)
Expand Down
Loading