From fbcc7fa46c8dce22a8aed30ce017762cfa7a841c Mon Sep 17 00:00:00 2001 From: Nikhitha Malkapuram Date: Tue, 11 Nov 2025 12:12:37 +0530 Subject: [PATCH 1/5] DSB fix for ubuntu 24.04 --- .../DeathStarBenchClientExecutorTests.cs | 21 ++++++++------- .../DeathStarBenchExecutorTests.cs | 7 ++--- .../DeathStarBench/DeathStarBenchExecutor.cs | 26 ++++++++++++++++--- .../DeathStarBenchServerExecutor.cs | 4 +-- .../profiles/PERF-NETWORK-DEATHSTARBENCH.json | 1 - 5 files changed, 40 insertions(+), 19 deletions(-) diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/DeathStarBench/DeathStarBenchClientExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/DeathStarBench/DeathStarBenchClientExecutorTests.cs index 326bcf4ad5..9307906816 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/DeathStarBench/DeathStarBenchClientExecutorTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/DeathStarBench/DeathStarBenchClientExecutorTests.cs @@ -148,9 +148,10 @@ public async Task DeathStarBenchClientExecutorExecutesExpectedCommands_SocialNet // exactly the same as what is executed. $"sudo bash {this.mockPackage.Path}/linux-x64/scripts/dockerComposeScript.sh", $"sudo chmod +x \"/usr/local/bin/docker-compose\"", - $"sudo python3 -m pip install -U pip", - $"sudo python3 -m pip install -U setuptools", - $"sudo -H python3 -m pip install aiohttp asyncio", + $"sudo python3 -m venv {this.mockPackage.Path}/linux-x64/venv", + $"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install -U pip", + $"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install -U setuptools", + $"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install aiohttp asyncio", $"sudo luarocks install luasocket", $"sudo bash {this.mockPackage.Path}/linux-x64/scripts/isSwarmNode.sh", $"sudo bash {this.mockPackage.Path}/linux-x64/scripts/isSwarmNode.sh", @@ -198,9 +199,10 @@ public async Task DeathStarBenchClientExecutorExecutesExpectedCommands_MediaMicr $"sudo chmod +x \"{this.mockPackage.Path}/linux-x64/mediamicroservices/wrk2/deps/luajit/src/luajit\"", $"sudo bash {this.mockPackage.Path}/linux-x64/scripts/dockerComposeScript.sh", $"sudo chmod +x \"/usr/local/bin/docker-compose\"", - $"sudo python3 -m pip install -U pip", - $"sudo python3 -m pip install -U setuptools", - $"sudo -H python3 -m pip install aiohttp asyncio", + $"sudo python3 -m venv {this.mockPackage.Path}/linux-x64/venv", + $"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install -U pip", + $"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install -U setuptools", + $"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install aiohttp asyncio", $"sudo luarocks install luasocket", $"sudo bash {this.mockPackage.Path}/linux-x64/scripts/isSwarmNode.sh", $"sudo bash {this.mockPackage.Path}/linux-x64/scripts/isSwarmNode.sh", @@ -243,9 +245,10 @@ public async Task DeathStarBenchClientExecutorExecutesExpectedCommands_HotelRese // exactly the same as what is executed. $"sudo bash {this.mockPackage.Path}/linux-x64/scripts/dockerComposeScript.sh", $"sudo chmod +x \"/usr/local/bin/docker-compose\"", - $"sudo python3 -m pip install -U pip", - $"sudo python3 -m pip install -U setuptools", - $"sudo -H python3 -m pip install aiohttp asyncio", + $"sudo python3 -m venv {this.mockPackage.Path}/linux-x64/venv", + $"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install -U pip", + $"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install -U setuptools", + $"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install aiohttp asyncio", $"sudo luarocks install luasocket", $"sudo bash {this.mockPackage.Path}/linux-x64/scripts/isSwarmNode.sh", $"sudo bash {this.mockPackage.Path}/linux-x64/scripts/isSwarmNode.sh", diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/DeathStarBench/DeathStarBenchExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/DeathStarBench/DeathStarBenchExecutorTests.cs index 643db87f79..53ad1cdbc9 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/DeathStarBench/DeathStarBenchExecutorTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/DeathStarBench/DeathStarBenchExecutorTests.cs @@ -96,9 +96,10 @@ public async Task DeathStarBenchExecutorInstallsDepedenciesAsExpected() { @"bash /home/user/tools/VirtualClient/packages/deathstarbench/linux-x64/scripts/dockerComposeScript.sh", @"chmod +x ""/usr/local/bin/docker-compose""", - @"python3 -m pip install -U pip", - @"python3 -m pip install -U setuptools", - @"-H python3 -m pip install aiohttp asyncio", + @"python3 -m venv /home/user/tools/VirtualClient/packages/deathstarbench/linux-x64/venv", + @"/home/user/tools/VirtualClient/packages/deathstarbench/linux-x64/venv/bin/pip install -U pip", + @"/home/user/tools/VirtualClient/packages/deathstarbench/linux-x64/venv/bin/pip install -U setuptools", + @"/home/user/tools/VirtualClient/packages/deathstarbench/linux-x64/venv/bin/pip install aiohttp asyncio", @"luarocks install luasocket", }; diff --git a/src/VirtualClient/VirtualClient.Actions/DeathStarBench/DeathStarBenchExecutor.cs b/src/VirtualClient/VirtualClient.Actions/DeathStarBench/DeathStarBenchExecutor.cs index d889dbb1cb..23e699144b 100644 --- a/src/VirtualClient/VirtualClient.Actions/DeathStarBench/DeathStarBenchExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/DeathStarBench/DeathStarBenchExecutor.cs @@ -124,6 +124,11 @@ public string ServiceName /// protected string ScriptsDirectory { get; set; } + /// + /// Path to Python executable in virtual environment. + /// + protected string PythonExecutablePath { get; set; } + /// /// An interface that can be used to communicate with the underlying system. /// @@ -546,16 +551,29 @@ await this.ExecuteCommandAsync(dockerComposeCommand, this.PackageDirectory, canc await this.SystemManager.MakeFileExecutableAsync(dockerComposeFilePath, this.Platform, cancellationToken) .ConfigureAwait(); - string pipUpgradeCommand = "python3 -m pip install -U pip"; + string installVenvCommand = "apt install python3-venv -y"; + await this.ExecuteCommandAsync(installVenvCommand, this.PackageDirectory, cancellationToken) + .ConfigureAwait(); + + // Create Python virtual environment to isolate dependencies + string venvPath = this.PlatformSpecifics.Combine(this.PackageDirectory, "venv"); + string createVenvCommand = $"python3 -m venv {venvPath}"; + await this.ExecuteCommandAsync(createVenvCommand, this.PackageDirectory, cancellationToken) + .ConfigureAwait(); + + this.PythonExecutablePath = this.PlatformSpecifics.Combine(venvPath, "bin", "python3"); + string venvPip = this.PlatformSpecifics.Combine(venvPath, "bin", "pip"); + + string pipUpgradeCommand = $"{venvPip} install -U pip"; await this.ExecuteCommandAsync(pipUpgradeCommand, this.PackageDirectory, cancellationToken) .ConfigureAwait(); - string setupToolsUpgradeCommand = "python3 -m pip install -U setuptools"; + string setupToolsUpgradeCommand = $"{venvPip} install -U setuptools"; await this.ExecuteCommandAsync(setupToolsUpgradeCommand, this.PackageDirectory, cancellationToken) .ConfigureAwait(); - // python3-pip installs pip3 and not pip, better to leave it on python which version of pip to use - string pipInstallPackagesCommand = "-H python3 -m pip install aiohttp asyncio"; + // Install required Python packages in virtual environment + string pipInstallPackagesCommand = $"{venvPip} install aiohttp asyncio"; await this.ExecuteCommandAsync(pipInstallPackagesCommand, this.PackageDirectory, cancellationToken) .ConfigureAwait(); diff --git a/src/VirtualClient/VirtualClient.Actions/DeathStarBench/DeathStarBenchServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/DeathStarBench/DeathStarBenchServerExecutor.cs index c4017f4ddd..79fca0da75 100644 --- a/src/VirtualClient/VirtualClient.Actions/DeathStarBench/DeathStarBenchServerExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/DeathStarBench/DeathStarBenchServerExecutor.cs @@ -123,12 +123,12 @@ await this.WaitAsync(DeathStarBenchExecutor.ServerWarmUpTime, cancellationToken) if (string.Equals(this.ServiceName, DeathStarBenchExecutor.MediaMicroservices, StringComparison.OrdinalIgnoreCase)) { - await this.ExecuteCommandAsync("python3 scripts/write_movie_info.py -c datasets/tmdb/casts.json -m datasets/tmdb/movies.json", this.ServiceDirectory, cancellationToken) + await this.ExecuteCommandAsync($"{this.PythonExecutablePath} scripts/write_movie_info.py -c datasets/tmdb/casts.json -m datasets/tmdb/movies.json", this.ServiceDirectory, cancellationToken) .ConfigureAwait(); } else if (string.Equals(this.ServiceName, DeathStarBenchExecutor.SocialNetwork, StringComparison.OrdinalIgnoreCase)) { - await this.ExecuteCommandAsync(@$"bash -c ""python3 scripts/init_social_graph.py --graph={this.GraphType} --limit=1000""", this.ServiceDirectory, cancellationToken) + await this.ExecuteCommandAsync(@$"bash -c ""{this.PythonExecutablePath} scripts/init_social_graph.py --graph={this.GraphType} --limit=1000""", this.ServiceDirectory, cancellationToken) .ConfigureAwait(); } } diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-NETWORK-DEATHSTARBENCH.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-NETWORK-DEATHSTARBENCH.json index 00e85425b8..3a78e093e9 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-NETWORK-DEATHSTARBENCH.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-NETWORK-DEATHSTARBENCH.json @@ -75,7 +75,6 @@ "Type": "LinuxPackageInstallation", "Parameters": { "Scenario": "InstallLinuxPackages", - "Repositories-Apt": "deb http://security.ubuntu.com/ubuntu xenial-security main", "Packages-Apt": "python3-pip,libssl-dev,libz-dev,luarocks,libssl1.0.0" } }, From 44bcf7147d3d075513355b03e09b2e0b1a16cb62 Mon Sep 17 00:00:00 2001 From: Nikhitha Malkapuram Date: Thu, 13 Nov 2025 15:34:47 +0530 Subject: [PATCH 2/5] fixing unit tests --- .../DeathStarBench/DeathStarBenchClientExecutorTests.cs | 3 +++ .../DeathStarBench/DeathStarBenchExecutorTests.cs | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/DeathStarBench/DeathStarBenchClientExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/DeathStarBench/DeathStarBenchClientExecutorTests.cs index 9307906816..fbdda16ed6 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/DeathStarBench/DeathStarBenchClientExecutorTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/DeathStarBench/DeathStarBenchClientExecutorTests.cs @@ -148,6 +148,7 @@ public async Task DeathStarBenchClientExecutorExecutesExpectedCommands_SocialNet // exactly the same as what is executed. $"sudo bash {this.mockPackage.Path}/linux-x64/scripts/dockerComposeScript.sh", $"sudo chmod +x \"/usr/local/bin/docker-compose\"", + $"sudo apt install python3-venv -y", $"sudo python3 -m venv {this.mockPackage.Path}/linux-x64/venv", $"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install -U pip", $"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install -U setuptools", @@ -199,6 +200,7 @@ public async Task DeathStarBenchClientExecutorExecutesExpectedCommands_MediaMicr $"sudo chmod +x \"{this.mockPackage.Path}/linux-x64/mediamicroservices/wrk2/deps/luajit/src/luajit\"", $"sudo bash {this.mockPackage.Path}/linux-x64/scripts/dockerComposeScript.sh", $"sudo chmod +x \"/usr/local/bin/docker-compose\"", + $"sudo apt install python3-venv -y", $"sudo python3 -m venv {this.mockPackage.Path}/linux-x64/venv", $"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install -U pip", $"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install -U setuptools", @@ -245,6 +247,7 @@ public async Task DeathStarBenchClientExecutorExecutesExpectedCommands_HotelRese // exactly the same as what is executed. $"sudo bash {this.mockPackage.Path}/linux-x64/scripts/dockerComposeScript.sh", $"sudo chmod +x \"/usr/local/bin/docker-compose\"", + $"sudo apt install python3-venv -y", $"sudo python3 -m venv {this.mockPackage.Path}/linux-x64/venv", $"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install -U pip", $"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install -U setuptools", diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/DeathStarBench/DeathStarBenchExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/DeathStarBench/DeathStarBenchExecutorTests.cs index 53ad1cdbc9..b95ac8efb9 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/DeathStarBench/DeathStarBenchExecutorTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/DeathStarBench/DeathStarBenchExecutorTests.cs @@ -96,6 +96,7 @@ public async Task DeathStarBenchExecutorInstallsDepedenciesAsExpected() { @"bash /home/user/tools/VirtualClient/packages/deathstarbench/linux-x64/scripts/dockerComposeScript.sh", @"chmod +x ""/usr/local/bin/docker-compose""", + @"apt install python3-venv -y", @"python3 -m venv /home/user/tools/VirtualClient/packages/deathstarbench/linux-x64/venv", @"/home/user/tools/VirtualClient/packages/deathstarbench/linux-x64/venv/bin/pip install -U pip", @"/home/user/tools/VirtualClient/packages/deathstarbench/linux-x64/venv/bin/pip install -U setuptools", @@ -117,7 +118,7 @@ public async Task DeathStarBenchExecutorInstallsDepedenciesAsExpected() await executor.OnInitialize(EventContext.None, CancellationToken.None); - Assert.AreEqual(6, executed); + Assert.AreEqual(8, executed); } } From a9d711f4a358ee34a33cbfc76aaef82d367fb77f Mon Sep 17 00:00:00 2001 From: Nikhitha Malkapuram Date: Thu, 13 Nov 2025 21:28:36 +0530 Subject: [PATCH 3/5] DSB support for ubuntu 24.04 --- .../profiles/PERF-NETWORK-DEATHSTARBENCH.json | 81 ++++++++++++------- 1 file changed, 50 insertions(+), 31 deletions(-) diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-NETWORK-DEATHSTARBENCH.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-NETWORK-DEATHSTARBENCH.json index 3a78e093e9..fb14074011 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-NETWORK-DEATHSTARBENCH.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-NETWORK-DEATHSTARBENCH.json @@ -54,35 +54,54 @@ } } ], - "Dependencies": [ - { - "Type": "DependencyPackageInstallation", - "Parameters": { - "Scenario": "InstallDeathStarBenchPackage", - "BlobContainer": "packages", - "BlobName": "deathstarbench.3.0.0.zip", - "PackageName": "deathstarbench", - "Extract": true - } - }, - { - "Type": "DockerInstallation", - "Parameters": { - "Scenario": "InstallDocker" - } - }, - { - "Type": "LinuxPackageInstallation", - "Parameters": { - "Scenario": "InstallLinuxPackages", - "Packages-Apt": "python3-pip,libssl-dev,libz-dev,luarocks,libssl1.0.0" - } - }, - { - "Type": "ApiServer", - "Parameters": { - "Scenario": "StartAPIServer" - } - } - ] + "Dependencies": [ + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallDeathStarBenchPackage", + "BlobContainer": "packages", + "BlobName": "deathstarbench.3.0.0.zip", + "PackageName": "deathstarbench", + "Extract": true + } + }, + { + "Type": "DockerInstallation", + "Parameters": { + "Scenario": "InstallDocker" + } + }, + { + "Type": "WgetPackageInstallation", + "Parameters": { + "Scenario": "InstallLibsslPackage", + "PackageName": "libssl", + "PackageUri": "http://security.ubuntu.com/ubuntu/pool/main/o/openssl1.0/libssl1.0.0_1.0.2n-1ubuntu5.13_amd64.deb", + "SupportedPlatforms": "linux-x64", + "Notes": "Example path to package -> /packages/libssl" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "InstallLibssl", + "SupportedPlatforms": "linux-x64", + "Command": "dpkg -i libssl1.0.0_1.0.2n-1ubuntu5.13_amd64.deb", + "WorkingDirectory": "{PackagePath:libssl}" + } + }, + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "python3-pip,libssl-dev,libz-dev,luarocks,libssl1.0.0,python3.12-venv" + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer" + } + } + ] } \ No newline at end of file From 0c0c07a4a8b64995ba12a31802635c0a3f18d0d5 Mon Sep 17 00:00:00 2001 From: Nikhitha Malkapuram Date: Thu, 13 Nov 2025 21:42:17 +0530 Subject: [PATCH 4/5] build fix --- .../DeathStarBench/DeathStarBenchClientProfileTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/DeathStarBench/DeathStarBenchClientProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/DeathStarBench/DeathStarBenchClientProfileTests.cs index e2975e5f72..3378049cdc 100644 --- a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/DeathStarBench/DeathStarBenchClientProfileTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/DeathStarBench/DeathStarBenchClientProfileTests.cs @@ -38,6 +38,7 @@ public void SetupFixture() DeathStarBenchExecutor.StateConfirmationPollingTimeout = TimeSpan.FromSeconds(1); this.mockFixture.SetupPackage("deathstarbench"); + this.mockFixture.SetupPackage("libssl"); this.mockFixture.SetupFile(@"/usr/local/bin/docker-compose"); } From 028e082ec6a872e07ab245350553b94ff913897d Mon Sep 17 00:00:00 2001 From: Nikhitha Malkapuram Date: Thu, 13 Nov 2025 22:27:04 +0530 Subject: [PATCH 5/5] functional tests fix --- .../DeathStarBenchServerProfileTests.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/DeathStarBench/DeathStarBenchServerProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/DeathStarBench/DeathStarBenchServerProfileTests.cs index 08e4e662c6..6dfacca21b 100644 --- a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/DeathStarBench/DeathStarBenchServerProfileTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/DeathStarBench/DeathStarBenchServerProfileTests.cs @@ -43,18 +43,19 @@ public async Task DeathStarBenchWorkloadProfileInstallsTheExpectedDependenciesOf new ClientInstance(this.serverAgentId, "1.2.3.5", "Server")); this.mockFixture.SystemManagement.SetupGet(sm => sm.AgentId).Returns(this.serverAgentId); + this.mockFixture.SetupPackage("libssl"); this.mockFixture.SetupFile(@"/usr/local/bin/docker-compose"); using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) - { + { executor.ExecuteActions = false; await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None) .ConfigureAwait(false); - - // Workload dependency package expectations - WorkloadAssert.WorkloadPackageInstalled(this.mockFixture, "deathstarbench"); } + + // Workload dependency package expectations + WorkloadAssert.WorkloadPackageInstalled(this.mockFixture, "deathstarbench"); } [Test] @@ -119,6 +120,8 @@ public void DeathStarBenchWorkloadProfileActionsWillNotBeExecutedIfTheClientWork new ClientInstance(this.clientAgentId, "1.2.3.4", "Client"), new ClientInstance(this.serverAgentId, "1.2.3.5", "Server")); + this.mockFixture.SetupPackage("libssl"); + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) { DependencyException error = Assert.ThrowsAsync(() => executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None));