diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/DeathStarBench/DeathStarBenchClientProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/DeathStarBench/DeathStarBenchClientProfileTests.cs index e2975e5f72..3378049cdc 100644 --- a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/DeathStarBench/DeathStarBenchClientProfileTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/DeathStarBench/DeathStarBenchClientProfileTests.cs @@ -38,6 +38,7 @@ public void SetupFixture() DeathStarBenchExecutor.StateConfirmationPollingTimeout = TimeSpan.FromSeconds(1); this.mockFixture.SetupPackage("deathstarbench"); + this.mockFixture.SetupPackage("libssl"); this.mockFixture.SetupFile(@"/usr/local/bin/docker-compose"); } diff --git a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/DeathStarBench/DeathStarBenchServerProfileTests.cs b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/DeathStarBench/DeathStarBenchServerProfileTests.cs index 08e4e662c6..6dfacca21b 100644 --- a/src/VirtualClient/VirtualClient.Actions.FunctionalTests/DeathStarBench/DeathStarBenchServerProfileTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.FunctionalTests/DeathStarBench/DeathStarBenchServerProfileTests.cs @@ -43,18 +43,19 @@ public async Task DeathStarBenchWorkloadProfileInstallsTheExpectedDependenciesOf new ClientInstance(this.serverAgentId, "1.2.3.5", "Server")); this.mockFixture.SystemManagement.SetupGet(sm => sm.AgentId).Returns(this.serverAgentId); + this.mockFixture.SetupPackage("libssl"); this.mockFixture.SetupFile(@"/usr/local/bin/docker-compose"); using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) - { + { executor.ExecuteActions = false; await executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None) .ConfigureAwait(false); - - // Workload dependency package expectations - WorkloadAssert.WorkloadPackageInstalled(this.mockFixture, "deathstarbench"); } + + // Workload dependency package expectations + WorkloadAssert.WorkloadPackageInstalled(this.mockFixture, "deathstarbench"); } [Test] @@ -119,6 +120,8 @@ public void DeathStarBenchWorkloadProfileActionsWillNotBeExecutedIfTheClientWork new ClientInstance(this.clientAgentId, "1.2.3.4", "Client"), new ClientInstance(this.serverAgentId, "1.2.3.5", "Server")); + this.mockFixture.SetupPackage("libssl"); + using (ProfileExecutor executor = TestDependencies.CreateProfileExecutor(profile, this.mockFixture.Dependencies)) { DependencyException error = Assert.ThrowsAsync(() => executor.ExecuteAsync(ProfileTiming.OneIteration(), CancellationToken.None)); diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/DeathStarBench/DeathStarBenchClientExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/DeathStarBench/DeathStarBenchClientExecutorTests.cs index 326bcf4ad5..fbdda16ed6 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/DeathStarBench/DeathStarBenchClientExecutorTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/DeathStarBench/DeathStarBenchClientExecutorTests.cs @@ -148,9 +148,11 @@ public async Task DeathStarBenchClientExecutorExecutesExpectedCommands_SocialNet // exactly the same as what is executed. $"sudo bash {this.mockPackage.Path}/linux-x64/scripts/dockerComposeScript.sh", $"sudo chmod +x \"/usr/local/bin/docker-compose\"", - $"sudo python3 -m pip install -U pip", - $"sudo python3 -m pip install -U setuptools", - $"sudo -H python3 -m pip install aiohttp asyncio", + $"sudo apt install python3-venv -y", + $"sudo python3 -m venv {this.mockPackage.Path}/linux-x64/venv", + $"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install -U pip", + $"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install -U setuptools", + $"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install aiohttp asyncio", $"sudo luarocks install luasocket", $"sudo bash {this.mockPackage.Path}/linux-x64/scripts/isSwarmNode.sh", $"sudo bash {this.mockPackage.Path}/linux-x64/scripts/isSwarmNode.sh", @@ -198,9 +200,11 @@ public async Task DeathStarBenchClientExecutorExecutesExpectedCommands_MediaMicr $"sudo chmod +x \"{this.mockPackage.Path}/linux-x64/mediamicroservices/wrk2/deps/luajit/src/luajit\"", $"sudo bash {this.mockPackage.Path}/linux-x64/scripts/dockerComposeScript.sh", $"sudo chmod +x \"/usr/local/bin/docker-compose\"", - $"sudo python3 -m pip install -U pip", - $"sudo python3 -m pip install -U setuptools", - $"sudo -H python3 -m pip install aiohttp asyncio", + $"sudo apt install python3-venv -y", + $"sudo python3 -m venv {this.mockPackage.Path}/linux-x64/venv", + $"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install -U pip", + $"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install -U setuptools", + $"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install aiohttp asyncio", $"sudo luarocks install luasocket", $"sudo bash {this.mockPackage.Path}/linux-x64/scripts/isSwarmNode.sh", $"sudo bash {this.mockPackage.Path}/linux-x64/scripts/isSwarmNode.sh", @@ -243,9 +247,11 @@ public async Task DeathStarBenchClientExecutorExecutesExpectedCommands_HotelRese // exactly the same as what is executed. $"sudo bash {this.mockPackage.Path}/linux-x64/scripts/dockerComposeScript.sh", $"sudo chmod +x \"/usr/local/bin/docker-compose\"", - $"sudo python3 -m pip install -U pip", - $"sudo python3 -m pip install -U setuptools", - $"sudo -H python3 -m pip install aiohttp asyncio", + $"sudo apt install python3-venv -y", + $"sudo python3 -m venv {this.mockPackage.Path}/linux-x64/venv", + $"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install -U pip", + $"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install -U setuptools", + $"sudo {this.mockPackage.Path}/linux-x64/venv/bin/pip install aiohttp asyncio", $"sudo luarocks install luasocket", $"sudo bash {this.mockPackage.Path}/linux-x64/scripts/isSwarmNode.sh", $"sudo bash {this.mockPackage.Path}/linux-x64/scripts/isSwarmNode.sh", diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/DeathStarBench/DeathStarBenchExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/DeathStarBench/DeathStarBenchExecutorTests.cs index 643db87f79..b95ac8efb9 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/DeathStarBench/DeathStarBenchExecutorTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/DeathStarBench/DeathStarBenchExecutorTests.cs @@ -96,9 +96,11 @@ public async Task DeathStarBenchExecutorInstallsDepedenciesAsExpected() { @"bash /home/user/tools/VirtualClient/packages/deathstarbench/linux-x64/scripts/dockerComposeScript.sh", @"chmod +x ""/usr/local/bin/docker-compose""", - @"python3 -m pip install -U pip", - @"python3 -m pip install -U setuptools", - @"-H python3 -m pip install aiohttp asyncio", + @"apt install python3-venv -y", + @"python3 -m venv /home/user/tools/VirtualClient/packages/deathstarbench/linux-x64/venv", + @"/home/user/tools/VirtualClient/packages/deathstarbench/linux-x64/venv/bin/pip install -U pip", + @"/home/user/tools/VirtualClient/packages/deathstarbench/linux-x64/venv/bin/pip install -U setuptools", + @"/home/user/tools/VirtualClient/packages/deathstarbench/linux-x64/venv/bin/pip install aiohttp asyncio", @"luarocks install luasocket", }; @@ -116,7 +118,7 @@ public async Task DeathStarBenchExecutorInstallsDepedenciesAsExpected() await executor.OnInitialize(EventContext.None, CancellationToken.None); - Assert.AreEqual(6, executed); + Assert.AreEqual(8, executed); } } diff --git a/src/VirtualClient/VirtualClient.Actions/DeathStarBench/DeathStarBenchExecutor.cs b/src/VirtualClient/VirtualClient.Actions/DeathStarBench/DeathStarBenchExecutor.cs index d889dbb1cb..23e699144b 100644 --- a/src/VirtualClient/VirtualClient.Actions/DeathStarBench/DeathStarBenchExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/DeathStarBench/DeathStarBenchExecutor.cs @@ -124,6 +124,11 @@ public string ServiceName /// protected string ScriptsDirectory { get; set; } + /// + /// Path to Python executable in virtual environment. + /// + protected string PythonExecutablePath { get; set; } + /// /// An interface that can be used to communicate with the underlying system. /// @@ -546,16 +551,29 @@ await this.ExecuteCommandAsync(dockerComposeCommand, this.PackageDirectory, canc await this.SystemManager.MakeFileExecutableAsync(dockerComposeFilePath, this.Platform, cancellationToken) .ConfigureAwait(); - string pipUpgradeCommand = "python3 -m pip install -U pip"; + string installVenvCommand = "apt install python3-venv -y"; + await this.ExecuteCommandAsync(installVenvCommand, this.PackageDirectory, cancellationToken) + .ConfigureAwait(); + + // Create Python virtual environment to isolate dependencies + string venvPath = this.PlatformSpecifics.Combine(this.PackageDirectory, "venv"); + string createVenvCommand = $"python3 -m venv {venvPath}"; + await this.ExecuteCommandAsync(createVenvCommand, this.PackageDirectory, cancellationToken) + .ConfigureAwait(); + + this.PythonExecutablePath = this.PlatformSpecifics.Combine(venvPath, "bin", "python3"); + string venvPip = this.PlatformSpecifics.Combine(venvPath, "bin", "pip"); + + string pipUpgradeCommand = $"{venvPip} install -U pip"; await this.ExecuteCommandAsync(pipUpgradeCommand, this.PackageDirectory, cancellationToken) .ConfigureAwait(); - string setupToolsUpgradeCommand = "python3 -m pip install -U setuptools"; + string setupToolsUpgradeCommand = $"{venvPip} install -U setuptools"; await this.ExecuteCommandAsync(setupToolsUpgradeCommand, this.PackageDirectory, cancellationToken) .ConfigureAwait(); - // python3-pip installs pip3 and not pip, better to leave it on python which version of pip to use - string pipInstallPackagesCommand = "-H python3 -m pip install aiohttp asyncio"; + // Install required Python packages in virtual environment + string pipInstallPackagesCommand = $"{venvPip} install aiohttp asyncio"; await this.ExecuteCommandAsync(pipInstallPackagesCommand, this.PackageDirectory, cancellationToken) .ConfigureAwait(); diff --git a/src/VirtualClient/VirtualClient.Actions/DeathStarBench/DeathStarBenchServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/DeathStarBench/DeathStarBenchServerExecutor.cs index c4017f4ddd..79fca0da75 100644 --- a/src/VirtualClient/VirtualClient.Actions/DeathStarBench/DeathStarBenchServerExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/DeathStarBench/DeathStarBenchServerExecutor.cs @@ -123,12 +123,12 @@ await this.WaitAsync(DeathStarBenchExecutor.ServerWarmUpTime, cancellationToken) if (string.Equals(this.ServiceName, DeathStarBenchExecutor.MediaMicroservices, StringComparison.OrdinalIgnoreCase)) { - await this.ExecuteCommandAsync("python3 scripts/write_movie_info.py -c datasets/tmdb/casts.json -m datasets/tmdb/movies.json", this.ServiceDirectory, cancellationToken) + await this.ExecuteCommandAsync($"{this.PythonExecutablePath} scripts/write_movie_info.py -c datasets/tmdb/casts.json -m datasets/tmdb/movies.json", this.ServiceDirectory, cancellationToken) .ConfigureAwait(); } else if (string.Equals(this.ServiceName, DeathStarBenchExecutor.SocialNetwork, StringComparison.OrdinalIgnoreCase)) { - await this.ExecuteCommandAsync(@$"bash -c ""python3 scripts/init_social_graph.py --graph={this.GraphType} --limit=1000""", this.ServiceDirectory, cancellationToken) + await this.ExecuteCommandAsync(@$"bash -c ""{this.PythonExecutablePath} scripts/init_social_graph.py --graph={this.GraphType} --limit=1000""", this.ServiceDirectory, cancellationToken) .ConfigureAwait(); } } diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-NETWORK-DEATHSTARBENCH.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-NETWORK-DEATHSTARBENCH.json index 00e85425b8..fb14074011 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-NETWORK-DEATHSTARBENCH.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-NETWORK-DEATHSTARBENCH.json @@ -54,36 +54,54 @@ } } ], - "Dependencies": [ - { - "Type": "DependencyPackageInstallation", - "Parameters": { - "Scenario": "InstallDeathStarBenchPackage", - "BlobContainer": "packages", - "BlobName": "deathstarbench.3.0.0.zip", - "PackageName": "deathstarbench", - "Extract": true - } - }, - { - "Type": "DockerInstallation", - "Parameters": { - "Scenario": "InstallDocker" - } - }, - { - "Type": "LinuxPackageInstallation", - "Parameters": { - "Scenario": "InstallLinuxPackages", - "Repositories-Apt": "deb http://security.ubuntu.com/ubuntu xenial-security main", - "Packages-Apt": "python3-pip,libssl-dev,libz-dev,luarocks,libssl1.0.0" - } - }, - { - "Type": "ApiServer", - "Parameters": { - "Scenario": "StartAPIServer" - } - } - ] + "Dependencies": [ + { + "Type": "DependencyPackageInstallation", + "Parameters": { + "Scenario": "InstallDeathStarBenchPackage", + "BlobContainer": "packages", + "BlobName": "deathstarbench.3.0.0.zip", + "PackageName": "deathstarbench", + "Extract": true + } + }, + { + "Type": "DockerInstallation", + "Parameters": { + "Scenario": "InstallDocker" + } + }, + { + "Type": "WgetPackageInstallation", + "Parameters": { + "Scenario": "InstallLibsslPackage", + "PackageName": "libssl", + "PackageUri": "http://security.ubuntu.com/ubuntu/pool/main/o/openssl1.0/libssl1.0.0_1.0.2n-1ubuntu5.13_amd64.deb", + "SupportedPlatforms": "linux-x64", + "Notes": "Example path to package -> /packages/libssl" + } + }, + { + "Type": "ExecuteCommand", + "Parameters": { + "Scenario": "InstallLibssl", + "SupportedPlatforms": "linux-x64", + "Command": "dpkg -i libssl1.0.0_1.0.2n-1ubuntu5.13_amd64.deb", + "WorkingDirectory": "{PackagePath:libssl}" + } + }, + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages-Apt": "python3-pip,libssl-dev,libz-dev,luarocks,libssl1.0.0,python3.12-venv" + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer" + } + } + ] } \ No newline at end of file