diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Prime95/Prime95ExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/Prime95/Prime95ExecutorTests.cs index f2e205c741..38ab753719 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/Prime95/Prime95ExecutorTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Prime95/Prime95ExecutorTests.cs @@ -167,12 +167,12 @@ public void Prime95ExecutorThrowsWhenTheWorkloadDoesNotProduceValidResults(Platf } [Test] - [TestCase(PlatformID.Win32NT, @"\win-x64\results.txt")] - [TestCase(PlatformID.Unix, @"/linux-x64/results.txt")] - public void Prime95ExecutorThrowsWhenWorkloadResultsFileNotFound(PlatformID platform, string resultsPath) + [TestCase(PlatformID.Win32NT)] + [TestCase(PlatformID.Unix)] + public void Prime95ExecutorThrowsWhenWorkloadResultsFileNotFound(PlatformID platform) { this.SetupTest(platform); - this.mockFixture.File.Setup(fe => fe.Exists(this.mockPackage.Path + resultsPath)) + this.mockFixture.File.Setup(fe => fe.Exists(It.Is(file => file.EndsWith("results.txt")))) .Returns(false); using (TestPrime95Executor executor = new TestPrime95Executor(this.mockFixture)) diff --git a/src/VirtualClient/VirtualClient.Actions/Prime95/Prime95Executor.cs b/src/VirtualClient/VirtualClient.Actions/Prime95/Prime95Executor.cs index 36ccb0048a..ebdb2025a1 100644 --- a/src/VirtualClient/VirtualClient.Actions/Prime95/Prime95Executor.cs +++ b/src/VirtualClient/VirtualClient.Actions/Prime95/Prime95Executor.cs @@ -12,6 +12,7 @@ namespace VirtualClient.Actions using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Http; + using Microsoft.AspNetCore.Http.HttpResults; using Microsoft.Extensions.DependencyInjection; using VirtualClient.Common; using VirtualClient.Common.Extensions; @@ -182,6 +183,11 @@ protected override async Task CleanupAsync(EventContext telemetryContext, Cancel processProxy.SafeKill(); } } + + if (this.fileSystem.File.Exists(this.ResultsFilePath)) + { + await this.fileSystem.File.DeleteAsync(this.ResultsFilePath, RetryPolicies.FileDelete); + } } /// @@ -221,8 +227,8 @@ protected override async Task InitializeAsync(EventContext telemetryContext, Can ErrorReason.DependencyNotFound); } - this.SettingsFilePath = this.Combine(this.Prime95Package.Path, "prime.txt"); - this.ResultsFilePath = this.Combine(this.Prime95Package.Path, "results.txt"); + this.SettingsFilePath = this.Combine(this.GetTempPath(), "prime.txt"); + this.ResultsFilePath = this.Combine(this.GetTempPath(), FileContext.GetFileName("results.txt", DateTime.UtcNow)); } /// @@ -298,10 +304,13 @@ await this.Logger.LogMessageAsync($"{this.TypeName}.ExecuteWorkload", telemetryC try { - if (this.fileSystem.File.Exists(this.ResultsFilePath)) + await RetryPolicies.FileOperations.ExecuteAsync(async () => { - results = await this.fileSystem.File.ReadAllTextAsync(this.ResultsFilePath); - } + if (this.fileSystem.File.Exists(this.ResultsFilePath)) + { + results = await this.fileSystem.File.ReadAllTextAsync(this.ResultsFilePath); + } + }); if (string.IsNullOrWhiteSpace(results)) { diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs index 87587facc3..a21c0d5894 100644 --- a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs +++ b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs @@ -5,9 +5,6 @@ namespace VirtualClient.Actions { using System; using System.Collections.Generic; - using System.ComponentModel; - using System.Linq; - using System.Net; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; diff --git a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs index 67d09aaba6..bd22c637d8 100644 --- a/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs +++ b/src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs @@ -42,11 +42,6 @@ public class SysbenchExecutor : VirtualClientComponent protected const string PythonCommand = "python3"; private readonly IStateManager stateManager; - private static readonly string[] SelectWorkloads = - { - "select_random_points", - "select_random_ranges" - }; /// /// Constructor for @@ -65,7 +60,7 @@ public SysbenchExecutor(IServiceCollection dependencies, IDictionary - /// The database name option passed to Sysbench. + /// The benchmark (e.g. OLTP, TPCC). /// public string Benchmark { @@ -111,7 +106,7 @@ public int? RecordCount } /// - /// The workload option passed to Sysbench. + /// The number of tables to create in the database. /// public int? TableCount { @@ -135,7 +130,7 @@ public int? Threads } /// - /// Number of records per table. + /// The database system (e.g. MySQL, PostgreSQL). /// public string DatabaseSystem { @@ -158,7 +153,7 @@ public string SuperUserPassword } /// - /// Number of records per table. + /// Number of warehouses. /// public int? WarehouseCount { @@ -170,7 +165,7 @@ public int? WarehouseCount } /// - /// The workload option passed to Sysbench. + /// The workload option passed to Sysbench (e.g. oltp_update_index, oltp_update_non_index). /// public string Workload { @@ -252,7 +247,6 @@ public static int GetRecordCount(ISystemManagement systemManagement, string data recordCountExponent = Math.Max(3, recordCountExponent); int recordEstimate = (int)Math.Pow(10, recordCountExponent); - int recordCount = records.GetValueOrDefault(recordEstimate); // record count specified in profile if it is the configurable scenario diff --git a/src/VirtualClient/VirtualClient.Contracts.UnitTests/PlatformSpecificsTests.cs b/src/VirtualClient/VirtualClient.Contracts.UnitTests/PlatformSpecificsTests.cs index 2991d2e58d..9bf4997f7f 100644 --- a/src/VirtualClient/VirtualClient.Contracts.UnitTests/PlatformSpecificsTests.cs +++ b/src/VirtualClient/VirtualClient.Contracts.UnitTests/PlatformSpecificsTests.cs @@ -63,6 +63,62 @@ public void GetPackagePathReturnsTheExpectedPathOnUnixSystems() Assert.AreEqual("/home/anyuser/virtualclient/packages/any.package/1.0.0", platformSpecifics.GetPackagePath("any.package", "1.0.0")); } + [Test] + public void GetLoggedInUserReturnsTheExpectedUserOnWindowsSystems() + { + PlatformSpecifics platformSpecifics = new TestPlatformSpecifics(PlatformID.Win32NT, Architecture.X64); + string user = platformSpecifics.GetLoggedInUser(); + Assert.AreEqual(Environment.UserName, user); + + platformSpecifics.SetEnvironmentVariable(EnvironmentVariable.SUDO_USER, "User01"); + } + + [Test] + public void GetLoggedInUserReturnsTheExpectedUserOnWindowsSystems_2() + { + // Environment variables do not matter on Windows and should not affect + // the return user. + PlatformSpecifics platformSpecifics = new TestPlatformSpecifics(PlatformID.Win32NT, Architecture.X64); + + platformSpecifics.SetEnvironmentVariable(EnvironmentVariable.SUDO_USER, "User01"); + string user = platformSpecifics.GetLoggedInUser(); + Assert.AreEqual(Environment.UserName, user); + + platformSpecifics.SetEnvironmentVariable(EnvironmentVariable.VC_SUDO_USER, "User02"); + user = platformSpecifics.GetLoggedInUser(); + Assert.AreEqual(Environment.UserName, user); + } + + [Test] + public void GetLoggedInUserReturnsTheExpectedUserOnUnixSystems() + { + PlatformSpecifics platformSpecifics = new TestPlatformSpecifics(PlatformID.Unix, Architecture.X64); + string user = platformSpecifics.GetLoggedInUser(); + Assert.AreEqual(Environment.UserName, user); + } + + [Test] + public void GetLoggedInUserReturnsTheExpectedUserOnUnixSystemsWhenSudoIsUsed() + { + PlatformSpecifics platformSpecifics = new TestPlatformSpecifics(PlatformID.Unix, Architecture.X64); + + string sudoUser = "User01"; + platformSpecifics.SetEnvironmentVariable(EnvironmentVariable.SUDO_USER, sudoUser); + string user = platformSpecifics.GetLoggedInUser(); + Assert.AreEqual(sudoUser, user); + } + + [Test] + public void GetLoggedInUserReturnsTheExpectedUserOnUnixSystemsWhenCustomSudoAlternativesAreUsed() + { + PlatformSpecifics platformSpecifics = new TestPlatformSpecifics(PlatformID.Unix, Architecture.X64); + + string sudoUser = "User01"; + platformSpecifics.SetEnvironmentVariable(EnvironmentVariable.VC_SUDO_USER, sudoUser); + string user = platformSpecifics.GetLoggedInUser(); + Assert.AreEqual(sudoUser, user); + } + [Test] public void GetPackagePathReturnsTheExpectedPathOnWindowsSystems() { diff --git a/src/VirtualClient/VirtualClient.Contracts/FileContext.cs b/src/VirtualClient/VirtualClient.Contracts/FileContext.cs index a986af1824..c0967ffb5f 100644 --- a/src/VirtualClient/VirtualClient.Contracts/FileContext.cs +++ b/src/VirtualClient/VirtualClient.Contracts/FileContext.cs @@ -16,6 +16,8 @@ namespace VirtualClient.Contracts /// public class FileContext { + private const string FileTimestampFormat = "yyyy-MM-ddTHH-mm-ss-fffffK"; + private static readonly Regex PathReservedCharacterExpression = new Regex(@"[""<>:|?*\\/]+", RegexOptions.Compiled); private static readonly Regex TemplatePlaceholderExpression = new Regex(@"\{(.*?)\}", RegexOptions.Compiled | RegexOptions.IgnoreCase); /// @@ -94,6 +96,19 @@ public FileContext(IFileInfo file, string contentType, string contentEncoding, s /// public string ToolName { get; } + /// + /// Returns a file name containing a timestamp as part of the name having removed any + /// characters not allowed in file paths (e.g. 2023-02-01T12-23-30241Z-randomwrite_4k_blocksize.log). + /// + /// The name of the file (e.g. randomwrite_4k_blocksize.log) + /// The timestamp to add to the file name. + public static string GetFileName(string fileName, DateTime timestamp) + { + return PathReservedCharacterExpression.Replace( + $"{timestamp.ToString(FileTimestampFormat)}-{fileName.RemoveWhitespace()}", + string.Empty); + } + /// /// Resolves placeholders in the path template provided. /// diff --git a/src/VirtualClient/VirtualClient.Contracts/FileUploadDescriptor.cs b/src/VirtualClient/VirtualClient.Contracts/FileUploadDescriptor.cs index bcc6b45aa5..7abe1acebf 100644 --- a/src/VirtualClient/VirtualClient.Contracts/FileUploadDescriptor.cs +++ b/src/VirtualClient/VirtualClient.Contracts/FileUploadDescriptor.cs @@ -24,9 +24,6 @@ public class FileUploadDescriptor /// The default extension for the file uploads. /// public const string UploadDescriptorFileExtension = "upload.json"; - - private const string FileTimestampFormat = "yyyy-MM-ddTHH-mm-ss-fffffK"; - private static readonly Regex PathReservedCharacterExpression = new Regex(@"[""<>:|?*\\/]+", RegexOptions.Compiled); private static readonly char[] PathDelimiters = new char[] { '/', '\\' }; /// @@ -173,19 +170,6 @@ public static IDictionary CreateManifest(FileContext fileC return fileManifest.ObscureSecrets(); } - /// - /// Returns a file name containing a timestamp as part of the name having removed any - /// characters not allowed in file paths (e.g. 2023-02-01T12-23-30241Z-randomwrite_4k_blocksize.log). - /// - /// The name of the file (e.g. randomwrite_4k_blocksize.log) - /// The timestamp to add to the file name. - public static string GetFileName(string fileName, DateTime timestamp) - { - return PathReservedCharacterExpression.Replace( - $"{timestamp.ToString(FileTimestampFormat)}-{fileName.RemoveWhitespace()}", - string.Empty); - } - /// /// Returns a for the current instance. /// diff --git a/src/VirtualClient/VirtualClient.Contracts/FileUploadDescriptorFactory.cs b/src/VirtualClient/VirtualClient.Contracts/FileUploadDescriptorFactory.cs index 4002c1c7fb..0674bd3bea 100644 --- a/src/VirtualClient/VirtualClient.Contracts/FileUploadDescriptorFactory.cs +++ b/src/VirtualClient/VirtualClient.Contracts/FileUploadDescriptorFactory.cs @@ -62,7 +62,7 @@ public static FileUploadDescriptor CreateDescriptor(FileContext fileContext, IDi if (timestamped) { - blobName = FileUploadDescriptor.GetFileName(blobName, fileContext.File.CreationTimeUtc); + blobName = FileContext.GetFileName(blobName, fileContext.File.CreationTimeUtc); } // The caller of this factory method makes the determination on the runtime parameters that are diff --git a/src/VirtualClient/VirtualClient.Contracts/PlatformSpecifics.cs b/src/VirtualClient/VirtualClient.Contracts/PlatformSpecifics.cs index c0b5ebccd4..6fb40811ed 100644 --- a/src/VirtualClient/VirtualClient.Contracts/PlatformSpecifics.cs +++ b/src/VirtualClient/VirtualClient.Contracts/PlatformSpecifics.cs @@ -453,18 +453,19 @@ public virtual string GetEnvironmentVariable(string variableName, EnvironmentVar public string GetLoggedInUser() { string loggedInUserName = Environment.UserName; - if (string.Equals(loggedInUserName, "root")) + if (this.Platform == PlatformID.Unix) { - loggedInUserName = this.GetEnvironmentVariable(EnvironmentVariable.SUDO_USER); - if (string.IsNullOrWhiteSpace(loggedInUserName)) + // Note that when the user is "root" and running a command with "sudo", there will be + // no "SUDO_USER" environment variable. + string sudoUser = this.GetEnvironmentVariable(EnvironmentVariable.SUDO_USER); + if (string.IsNullOrWhiteSpace(sudoUser)) { - loggedInUserName = this.GetEnvironmentVariable(EnvironmentVariable.VC_SUDO_USER); - if (string.IsNullOrEmpty(loggedInUserName)) - { - // When the user is "root" and running a command with "sudo", there will be - // no "SUDO_USER" environment variable. - return "root"; - } + sudoUser = this.GetEnvironmentVariable(EnvironmentVariable.VC_SUDO_USER); + } + + if (!string.IsNullOrEmpty(sudoUser)) + { + return sudoUser; } } diff --git a/src/VirtualClient/VirtualClient.Contracts/VirtualClientComponentExtensions.cs b/src/VirtualClient/VirtualClient.Contracts/VirtualClientComponentExtensions.cs index 9417f3b634..b19412975a 100644 --- a/src/VirtualClient/VirtualClient.Contracts/VirtualClientComponentExtensions.cs +++ b/src/VirtualClient/VirtualClient.Contracts/VirtualClientComponentExtensions.cs @@ -378,7 +378,7 @@ public static async Task RequestFileUploadAsync(this VirtualClientComponent comp fileSystem.Directory.CreateDirectory(targetDirectory); } - string fileName = FileUploadDescriptor.GetFileName(FileUploadDescriptor.UploadDescriptorFileExtension, DateTime.UtcNow); + string fileName = FileContext.GetFileName(FileUploadDescriptor.UploadDescriptorFileExtension, DateTime.UtcNow); string filePath = component.Combine(targetDirectory, fileName.ToLowerInvariant()); await fileSystem.File.WriteAllTextAsync(filePath, descriptor.ToJson()); diff --git a/src/VirtualClient/VirtualClient.Core/Logging/Console/ConsoleLogger.cs b/src/VirtualClient/VirtualClient.Core/Logging/Console/ConsoleLogger.cs index 236499b361..b8127f326e 100644 --- a/src/VirtualClient/VirtualClient.Core/Logging/Console/ConsoleLogger.cs +++ b/src/VirtualClient/VirtualClient.Core/Logging/Console/ConsoleLogger.cs @@ -41,7 +41,7 @@ public ConsoleLogger(string categoryName, LogLevel minimumLogLevel = LogLevel.In /// /// The default console logger. /// - public static ConsoleLogger Default { get; set; } = new ConsoleLogger("VirtualClient", LogLevel.Debug); + public static ConsoleLogger Default { get; set; } = new ConsoleLogger("VirtualClient", LogLevel.Trace); /// /// True to include log severity levels in output. diff --git a/src/VirtualClient/VirtualClient.Core/SystemManagementExtensions.cs b/src/VirtualClient/VirtualClient.Core/SystemManagementExtensions.cs index 3d20904e05..ab1b6a681f 100644 --- a/src/VirtualClient/VirtualClient.Core/SystemManagementExtensions.cs +++ b/src/VirtualClient/VirtualClient.Core/SystemManagementExtensions.cs @@ -10,13 +10,17 @@ namespace VirtualClient using System.Net.NetworkInformation; using System.Threading; using System.Threading.Tasks; + using Microsoft.CodeAnalysis.CSharp.Syntax; + using Microsoft.Extensions.Logging; using Microsoft.Win32; using Newtonsoft.Json.Linq; using Polly; using VirtualClient.Common; using VirtualClient.Common.Contracts; using VirtualClient.Common.Extensions; + using VirtualClient.Common.Telemetry; using VirtualClient.Contracts; + using VirtualClient.Logging; /// /// Extension methods for instances. @@ -179,5 +183,70 @@ await rebootSystem.StartAndWaitAsync(cancellationToken) } } } + + /// + /// Sets the directory (and any subdirectories or files) to allow full permissions on the system to any + /// user or group (e.g. chmod -R 777 on Linux). + /// + /// https://linuxhandbook.com/linux-file-permissions/ + /// https://chmodcommand.com/chmod-777/ + /// + /// + /// The system management instance. + /// The path to the directory. + /// The OS platform on which the binary should be executable. + /// Provides context information for telemetry events. + /// A token that can be used to cancel the operation. + /// Defines a user to apply to the directory structure as owner. + /// A logger to use for capturing telemetry. + public static async Task SetFullPermissionsAsync( + this ISystemManagement systemManagement, string directoryPath, PlatformID platform, EventContext telemetryContext, CancellationToken cancellationToken, string owner = null, ILogger logger = null) + { + systemManagement.ThrowIfNull(nameof(systemManagement)); + directoryPath.ThrowIfNullOrWhiteSpace(nameof(directoryPath)); + PlatformSpecifics.ThrowIfNotSupported(platform); + + if (!systemManagement.FileSystem.Directory.Exists(directoryPath)) + { + throw new DependencyException($"The directory '{directoryPath}' does not exist.", ErrorReason.WorkloadDependencyMissing); + } + + switch (platform) + { + case PlatformID.Unix: + // https://chmodcommand.com/chmod-777/ + // https://linuxhandbook.com/linux-file-permissions/ + + EventContext relatedContext = telemetryContext.Clone(); + relatedContext.AddContext("directory", directoryPath); + relatedContext.AddContext("owner", owner); + relatedContext.AddContext("permissions", "777"); + + logger?.LogMessage($"SetFullPermissions (directory = '{directoryPath}', owner = '{owner}')", relatedContext); + + using (IProcessProxy chmod = systemManagement.ProcessManager.CreateProcess("sudo", $"chmod -R 777 \"{directoryPath}\"")) + { + await chmod.StartAndWaitAsync(cancellationToken, TimeSpan.FromSeconds(30)); + + chmod.ThrowIfErrored( + ProcessProxy.DefaultSuccessCodes, + $"Failed to attribute the directory '{directoryPath}' with full permissions."); + } + + if (!string.IsNullOrWhiteSpace(owner)) + { + using (IProcessProxy chown = systemManagement.ProcessManager.CreateProcess("sudo", $"chown {owner}:{owner} \"{directoryPath}\"")) + { + await chown.StartAndWaitAsync(cancellationToken, TimeSpan.FromSeconds(30)); + + chown.ThrowIfErrored( + ProcessProxy.DefaultSuccessCodes, + $"Failed to set owner for the directory '{directoryPath}' to '{owner}'."); + } + } + + break; + } + } } } diff --git a/src/VirtualClient/VirtualClient.Core/UnixDiskManager.cs b/src/VirtualClient/VirtualClient.Core/UnixDiskManager.cs index 147acd11dd..f6d5608efe 100644 --- a/src/VirtualClient/VirtualClient.Core/UnixDiskManager.cs +++ b/src/VirtualClient/VirtualClient.Core/UnixDiskManager.cs @@ -9,7 +9,6 @@ namespace VirtualClient using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; - using Org.BouncyCastle.Pqc.Crypto.Lms; using Polly; using VirtualClient.Common; using VirtualClient.Common.Extensions; @@ -21,33 +20,6 @@ namespace VirtualClient /// public class UnixDiskManager : DiskManager { - private const string BusInfo = "businfo"; - private const string Capacity = "capacity"; - private const string Capabilities = "capabilities"; - private const string Claimed = "claimed"; - private const string Class = "class"; - private const string Device = "dev"; - private const string Description = "description"; - private const string FileSystem = "filesystem"; - private const string Handle = "handle"; - private const string Id = "id"; - private const string LogicalName = "logicalname"; - private const string PhysicalId = "physid"; - private const string Product = "product"; - private const string Serial = "serial"; - private const string Size = "size"; - private const string Vendor = "vendor"; - private const string Version = "version"; - - // Key = The mount point/path - // Value = The relative priority in relation to other paths or mount points when accessing the disk. < 0 = not typically accessible. - private static readonly Dictionary SystemDefinedMountPoints = new Dictionary(StringComparer.OrdinalIgnoreCase) - { - { "/", 100 }, - { "/mnt", 100 }, - { "/boot/efi", -1 } // Not typically accessible - }; - /// /// Initializes a new instance of the class. /// diff --git a/src/VirtualClient/VirtualClient.Dependencies.UnitTests/MountDisksTests.cs b/src/VirtualClient/VirtualClient.Dependencies.UnitTests/MountDisksTests.cs index d4675269bf..bd36d49138 100644 --- a/src/VirtualClient/VirtualClient.Dependencies.UnitTests/MountDisksTests.cs +++ b/src/VirtualClient/VirtualClient.Dependencies.UnitTests/MountDisksTests.cs @@ -37,9 +37,11 @@ public async Task MountDisksMountsTheExpectedPathOnUnix() foreach (DiskVolume diskVolume in this.diskVolumes) { - string expectedMountDirectory = this.mockFixture.StandardizePath(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile)); - string expectedMountPoint = this.mockFixture.Combine(expectedMountDirectory, diskVolume.GetDefaultMountPointName()); - + // e.g. + // /home/user/mnt_dev_sdc1 + // /home/user/mnt_dev_sdd1 + // /home/user/mnt_dev_sdd2 + string expectedMountPoint = $"/home/{Environment.UserName}/{diskVolume.GetDefaultMountPointName()}"; this.mockFixture.DiskManager.Verify(mgr => mgr.CreateMountPointAsync(diskVolume, expectedMountPoint, It.IsAny())); } } @@ -58,42 +60,84 @@ public async Task MountDisksMountsTheExpectedPathOnUnixWhenMultipleVolumesArePre foreach (DiskVolume diskVolume in this.diskVolumes) { - string expectedMountDirectory = this.mockFixture.StandardizePath(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile)); - string expectedMountPoint = this.mockFixture.Combine(expectedMountDirectory, diskVolume.GetDefaultMountPointName()); + string expectedMountPoint = $"/home/{Environment.UserName}/{diskVolume.GetDefaultMountPointName()}"; + this.mockFixture.DiskManager.Verify(mgr => mgr.CreateMountPointAsync(diskVolume, expectedMountPoint, It.IsAny())); + } + } + } + [Test] + public async Task MountDisksHandlesCasesWhenRunningOnUnixWithSudo() + { + this.SetupTest(PlatformID.Unix); + + using (MountDisks diskMounter = new MountDisks(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + diskMounter.PlatformSpecifics.SetEnvironmentVariable(EnvironmentVariable.SUDO_USER, "user01"); + await diskMounter.ExecuteAsync(CancellationToken.None); + + foreach (DiskVolume diskVolume in this.diskVolumes) + { + // e.g. + // /home/user01/mnt_dev_sdc1 + // /home/user01/mnt_dev_sdd1 + // /home/user01/mnt_dev_sdd2 + string expectedMountPoint = $"/home/user01/{diskVolume.GetDefaultMountPointName()}"; this.mockFixture.DiskManager.Verify(mgr => mgr.CreateMountPointAsync(diskVolume, expectedMountPoint, It.IsAny())); } } } [Test] - public async Task MountDisksMountsTheExpectedPathOnUnixWhenAMountPrefixIsProvided() + public async Task MountDisksHandlesCasesWhenRunningOnUnixAsRoot() { this.SetupTest(PlatformID.Unix); - this.mockFixture.Parameters["MountPointPrefix"] = "mnt_test"; using (MountDisks diskMounter = new MountDisks(this.mockFixture.Dependencies, this.mockFixture.Parameters)) { + // SUDO_USER will be set to "root" when logged in as root. + diskMounter.PlatformSpecifics.SetEnvironmentVariable(EnvironmentVariable.SUDO_USER, "root"); await diskMounter.ExecuteAsync(CancellationToken.None); foreach (DiskVolume diskVolume in this.diskVolumes) { - string expectedMountDirectory = this.mockFixture.StandardizePath(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile)); + // e.g. + // /mnt_dev_sdc1 + // /mnt_dev_sdd1 + // /mnt_dev_sdd2 + string expectedMountPoint = $"/{diskVolume.GetDefaultMountPointName()}"; + this.mockFixture.DiskManager.Verify(mgr => mgr.CreateMountPointAsync(diskVolume, expectedMountPoint, It.IsAny())); + } + } + } + + [Test] + [TestCase("mount_points")] + [TestCase("/mount_points")] + [TestCase("/mount_points/")] + [TestCase(" /mount_points/ ")] + public async Task MountDisksMountsTheExpectedPathOnUnixWhenAMountLocationIsProvided(string expectedMountLocation) + { + this.SetupTest(PlatformID.Unix); + this.mockFixture.Parameters["MountLocation"] = expectedMountLocation; - string expectedMountPoint = this.mockFixture.Combine( - expectedMountDirectory, - diskVolume.GetDefaultMountPointName(prefix: "mnt_test")); + using (MountDisks diskMounter = new MountDisks(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await diskMounter.ExecuteAsync(CancellationToken.None); + foreach (DiskVolume diskVolume in this.diskVolumes) + { + string expectedMountPoint = $"/{expectedMountLocation.Trim().Trim('/')}/{diskVolume.GetDefaultMountPointName()}"; this.mockFixture.DiskManager.Verify(mgr => mgr.CreateMountPointAsync(diskVolume, expectedMountPoint, It.IsAny())); } } } [Test] - public async Task MountDisksMountsTheExpectedPathOnUnixWhenAMountLocationIsProvided_Root() + public async Task MountDisksMountsTheExpectedPathOnUnixWhenAMountPrefixIsProvided() { this.SetupTest(PlatformID.Unix); - this.mockFixture.Parameters["MountLocation"] = "Root"; + this.mockFixture.Parameters["MountPointPrefix"] = "mnt_test"; using (MountDisks diskMounter = new MountDisks(this.mockFixture.Dependencies, this.mockFixture.Parameters)) { @@ -101,12 +145,52 @@ public async Task MountDisksMountsTheExpectedPathOnUnixWhenAMountLocationIsProvi foreach (DiskVolume diskVolume in this.diskVolumes) { - string expectedMountPoint = this.mockFixture.Combine("/", diskVolume.GetDefaultMountPointName()); + string expectedMountPoint = $"/home/{Environment.UserName}/{diskVolume.GetDefaultMountPointName(prefix: "mnt_test")}"; this.mockFixture.DiskManager.Verify(mgr => mgr.CreateMountPointAsync(diskVolume, expectedMountPoint, It.IsAny())); } } } + [Test] + public async Task MountDisksSetsExpectedPermissionsOnTheMountPointDirectoryOnUnixSystems() + { + this.SetupTest(PlatformID.Unix); + + using (MountDisks diskMounter = new MountDisks(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await diskMounter.ExecuteAsync(CancellationToken.None); + + foreach (DiskVolume diskVolume in this.diskVolumes) + { + string expectedUser = Environment.UserName; + string expectedDirectory = $"/home/{Environment.UserName}/{diskVolume.GetDefaultMountPointName()}"; + Assert.IsTrue(this.mockFixture.ProcessManager.CommandsExecuted($"chmod -R 777 \"{expectedDirectory}\"")); + Assert.IsTrue(this.mockFixture.ProcessManager.CommandsExecuted($"chown {expectedUser}:{expectedUser} \"{expectedDirectory}\"")); + } + } + } + + [Test] + public async Task MountDisksSetsExpectedPermissionsOnTheMountPointDirectoryOnUnixSystemsWhenRunningAsRoot() + { + this.SetupTest(PlatformID.Unix); + + using (MountDisks diskMounter = new MountDisks(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + // SUDO_USER will be set to "root" when logged in as root. + diskMounter.PlatformSpecifics.SetEnvironmentVariable(EnvironmentVariable.SUDO_USER, "root"); + await diskMounter.ExecuteAsync(CancellationToken.None); + + foreach (DiskVolume diskVolume in this.diskVolumes) + { + string expectedUser = "root"; + string expectedDirectory = $"/{diskVolume.GetDefaultMountPointName()}"; + Assert.IsTrue(this.mockFixture.ProcessManager.CommandsExecuted($"chmod -R 777 \"{expectedDirectory}\"")); + Assert.IsTrue(this.mockFixture.ProcessManager.CommandsExecuted($"chown {expectedUser}:{expectedUser} \"{expectedDirectory}\"")); + } + } + } + [Test] public async Task MountDisksMountsTheExpectedPathOnWindows() { @@ -118,9 +202,7 @@ public async Task MountDisksMountsTheExpectedPathOnWindows() foreach (DiskVolume diskVolume in this.diskVolumes) { - string expectedMountDirectory = this.mockFixture.StandardizePath(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile)); - string expectedMountPoint = this.mockFixture.Combine(expectedMountDirectory, diskVolume.GetDefaultMountPointName()); - + string expectedMountPoint = this.mockFixture.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), diskVolume.GetDefaultMountPointName()); this.mockFixture.DiskManager.Verify(mgr => mgr.CreateMountPointAsync(diskVolume, expectedMountPoint, It.IsAny())); } } @@ -139,9 +221,28 @@ public async Task MountDisksMountsTheExpectedPathOnWindowsWhenMultipleVolumesAre foreach (DiskVolume diskVolume in this.diskVolumes) { - string expectedMountDirectory = this.mockFixture.StandardizePath(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile)); - string expectedMountPoint = this.mockFixture.Combine(expectedMountDirectory, diskVolume.GetDefaultMountPointName()); + string expectedMountPoint = this.mockFixture.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), diskVolume.GetDefaultMountPointName()); + this.mockFixture.DiskManager.Verify(mgr => mgr.CreateMountPointAsync(diskVolume, expectedMountPoint, It.IsAny())); + } + } + } + [Test] + [TestCase("C:\\mount_points")] + [TestCase("C:\\mount_points\\")] + [TestCase(" C:\\mount_points\\ ")] + public async Task MountDisksMountsTheExpectedPathOnWindowsWhenAMountLocationIsProvided(string expectedMountLocation) + { + this.SetupTest(PlatformID.Win32NT); + this.mockFixture.Parameters["MountLocation"] = expectedMountLocation; + + using (MountDisks diskMounter = new MountDisks(this.mockFixture.Dependencies, this.mockFixture.Parameters)) + { + await diskMounter.ExecuteAsync(CancellationToken.None); + + foreach (DiskVolume diskVolume in this.diskVolumes) + { + string expectedMountPoint = $@"{expectedMountLocation.Trim().Trim('\\')}\{diskVolume.GetDefaultMountPointName()}"; this.mockFixture.DiskManager.Verify(mgr => mgr.CreateMountPointAsync(diskVolume, expectedMountPoint, It.IsAny())); } } @@ -159,9 +260,7 @@ public async Task MountDisksMountsTheExpectedPathOnWindowsWhenAMountPrefixIsProv foreach (DiskVolume diskVolume in this.diskVolumes) { - string expectedMountDirectory = this.mockFixture.StandardizePath(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile)); - string expectedMountPoint = this.mockFixture.Combine(expectedMountDirectory, diskVolume.GetDefaultMountPointName(prefix: "mnt_test")); - + string expectedMountPoint = this.mockFixture.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), diskVolume.GetDefaultMountPointName(prefix: "mnt_test")); this.mockFixture.DiskManager.Verify(mgr => mgr.CreateMountPointAsync(diskVolume, expectedMountPoint, It.IsAny())); } } diff --git a/src/VirtualClient/VirtualClient.Dependencies.UnitTests/MySqlServer/MySQLServerConfigurationTests.cs b/src/VirtualClient/VirtualClient.Dependencies.UnitTests/MySqlServer/MySQLServerConfigurationTests.cs index 85afff46d6..7fd5efbea7 100644 --- a/src/VirtualClient/VirtualClient.Dependencies.UnitTests/MySqlServer/MySQLServerConfigurationTests.cs +++ b/src/VirtualClient/VirtualClient.Dependencies.UnitTests/MySqlServer/MySQLServerConfigurationTests.cs @@ -58,7 +58,7 @@ public async Task MySQLConfigurationExecutesTheExpectedProcessForConfigureServer string[] expectedCommands = { - $"python3 {this.packagePath}/configure.py --serverIp 1.2.3.4 --innoDbDirs \"/home/user/mnt_dev_sdc1;/home/user/mnt_dev_sdd1;/home/user/mnt_dev_sde1;\"", + $"python3 {this.packagePath}/configure.py --serverIp 1.2.3.4 --innoDbDirs \"/home/user/mnt_dev_sdc1/mysql;/home/user/mnt_dev_sdd1/mysql;/home/user/mnt_dev_sde1/mysql;\"", }; int commandNumber = 0; @@ -115,7 +115,7 @@ public async Task MySQLConfigurationExecutesTheExpectedProcessForConfigureServer string[] expectedCommands = { - $"python3 {this.packagePath}/configure.py --serverIp 1.2.3.4 --innoDbDirs \"/home/user/mnt_dev_sdc1;/home/user/mnt_dev_sdd1;/home/user/mnt_dev_sde1;\" " + + $"python3 {this.packagePath}/configure.py --serverIp 1.2.3.4 --innoDbDirs \"/home/user/mnt_dev_sdc1/mysql;/home/user/mnt_dev_sdd1/mysql;/home/user/mnt_dev_sde1/mysql;\" " + $"--inMemory 8192", }; @@ -326,7 +326,7 @@ public async Task MySQLConfigurationExecutesTheExpectedProcessForDistributeDatab string[] expectedCommands = { - $"python3 {this.packagePath}/distribute-database.py --dbName mysql-test --directories \"/home/user/mnt_dev_sdc1;/home/user/mnt_dev_sdd1;/home/user/mnt_dev_sde1;\"", + $"python3 {this.packagePath}/distribute-database.py --dbName mysql-test --directories \"/home/user/mnt_dev_sdc1/mysql;/home/user/mnt_dev_sdd1/mysql;/home/user/mnt_dev_sde1/mysql;\"", }; int commandNumber = 0; diff --git a/src/VirtualClient/VirtualClient.Dependencies.UnitTests/PostgreSQLServer/PostgreSQLServerConfigurationTests.cs b/src/VirtualClient/VirtualClient.Dependencies.UnitTests/PostgreSQLServer/PostgreSQLServerConfigurationTests.cs index 5f2b0d6a4e..73663a6427 100644 --- a/src/VirtualClient/VirtualClient.Dependencies.UnitTests/PostgreSQLServer/PostgreSQLServerConfigurationTests.cs +++ b/src/VirtualClient/VirtualClient.Dependencies.UnitTests/PostgreSQLServer/PostgreSQLServerConfigurationTests.cs @@ -185,12 +185,16 @@ public async Task PostgreSQLServerConfigurationExecutesTheExpectedProcessForDist if (platform == PlatformID.Unix) { - expectedCommand = $"python3 {this.packagePath}/distribute-database.py --dbName hammerdbtest --directories \"/home/user/mnt_dev_sdc1;/home/user/mnt_dev_sdd1;/home/user/mnt_dev_sde1;\" --password [A-Za-z0-9+/=]+"; + expectedCommand = + $"python3 {this.packagePath}/distribute-database.py " + + $"--dbName hammerdbtest " + + $"--directories \"/home/user/mnt_dev_sdc1/postgresql;/home/user/mnt_dev_sdd1/postgresql;/home/user/mnt_dev_sde1/postgresql;\" " + + $"--password [A-Za-z0-9+/=]+"; } else { string tempPackagePath = this.packagePath.Replace(@"\", @"\\"); - expectedCommand = $"python3 {tempPackagePath}/distribute-database.py --dbName hammerdbtest --directories \"D:\\\\;E:\\\\;F:\\\\;\" --password [A-Za-z0-9+/=]+"; + expectedCommand = $"python3 {tempPackagePath}/distribute-database.py --dbName hammerdbtest --directories \"D:\\\\postgresql;E:\\\\postgresql;F:\\\\postgresql;\" --password [A-Za-z0-9+/=]+"; } this.mockFixture.ProcessManager.OnCreateProcess = (exe, arguments, workingDir) => diff --git a/src/VirtualClient/VirtualClient.Dependencies/MountDisks.cs b/src/VirtualClient/VirtualClient.Dependencies/MountDisks.cs index 1f52ebfb6f..cb2daadf0e 100644 --- a/src/VirtualClient/VirtualClient.Dependencies/MountDisks.cs +++ b/src/VirtualClient/VirtualClient.Dependencies/MountDisks.cs @@ -99,13 +99,7 @@ protected override async Task ExecuteAsync(EventContext telemetryContext, Cancel ErrorReason.DependencyNotFound); } - string mountLocation = null; - if (string.Equals(this.MountLocation, "Root") && this.Platform == PlatformID.Unix) - { - mountLocation = $"/"; - } - - if (await this.CreateMountPointsAsync(disks, this.MountPointPrefix, mountLocation, cancellationToken)) + if (await this.CreateMountPointsAsync(disks, telemetryContext, this.MountPointPrefix, this.MountLocation, cancellationToken)) { // Refresh the disks to pickup the mount point changes. await Task.Delay(1000); @@ -125,15 +119,7 @@ protected override async Task ExecuteAsync(EventContext telemetryContext, Cancel } } - /// - /// Creates mount points for any disks that do not have them already. - /// - /// This disks for which mount points need to be created. - /// A token that can be used to cancel the operation. - /// The prefix to use for the mount points (e.g. mnt_vc). - /// The parent directory in which the mount points will be created. Default is the user home/profile directory. - /// True if any 1 or more mount points are created, false if none. - private async Task CreateMountPointsAsync(IEnumerable disks, string mountPrefix = null, string mountDirectory = null, CancellationToken cancellationToken = default(CancellationToken)) + private async Task CreateMountPointsAsync(IEnumerable disks, EventContext telemetryContext, string mountPrefix = null, string mountDirectory = null, CancellationToken cancellationToken = default(CancellationToken)) { bool mountPointsCreated = false; @@ -149,30 +135,79 @@ protected override async Task ExecuteAsync(EventContext telemetryContext, Cancel foreach (Disk disk in disks.Where(d => !d.IsOperatingSystem())) { IEnumerable diskVolumes = disk.Volumes.Where(v => v.AccessPaths?.Any() != true); - + if (diskVolumes?.Any() == true) { // mount every volume that doesn't have an accessPath. - foreach (DiskVolume diskVolume in disk.Volumes.Where(v => v.AccessPaths?.Any() != true)) + foreach (DiskVolume volume in disk.Volumes.Where(v => v.AccessPaths?.Any() != true)) { string newMountPoint = null; - string mountPointName = diskVolume.GetDefaultMountPointName(prefix: mountPrefix); - string mountPointPath = mountDirectory ?? Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); - - // e.g. - // C:\Users\User\mnt_c - // C:\Users\User\mnt_d - // /home/user/mnt_dev_sdc1 - // /home/user/mnt_dev_sdd1 - // /home/user/mnt_dev_sdd2 + string mountPointName = volume.GetDefaultMountPointName(prefix: mountPrefix); + string mountPointPath = mountDirectory?.Trim(); + + if (string.IsNullOrWhiteSpace(mountPointPath)) + { + switch (this.Platform) + { + case PlatformID.Unix: + string user = this.PlatformSpecifics.GetLoggedInUser(); + if (string.Equals(user, "root")) + { + // When running as root: + // /mnt_dev_sdc1 + // /mnt_dev_sdd1 + mountPointPath = "/"; + } + else + { + // e.g. + // When running as a given user (including when sudo is used): + // /home/user/mnt_dev_sdc1 + // /home/user/mnt_dev_sdd1 + // /home/user/mnt_dev_sdd2 + mountPointPath = $"/home/{user}"; + } + + break; + + case PlatformID.Win32NT: + // e.g. + // C:\Users\User\mnt_c + // C:\Users\User\mnt_d + mountPointPath = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); + break; + } + } + + if (this.Platform == PlatformID.Unix && !mountPointPath.StartsWith("/")) + { + mountPointPath = $"/{mountPointPath}"; + } + newMountPoint = this.Combine(mountPointPath, mountPointName); - if (!this.systemManager.FileSystem.Directory.Exists(newMountPoint)) + if (!this.fileSystem.Directory.Exists(newMountPoint)) { - this.systemManager.FileSystem.Directory.CreateDirectory(newMountPoint).Create(); + this.fileSystem.Directory.CreateDirectory(newMountPoint); } - await this.systemManager.DiskManager.CreateMountPointAsync(diskVolume, newMountPoint, cancellationToken); + await this.diskManager.CreateMountPointAsync( + volume, + newMountPoint, + CancellationToken.None); + + // We want the mount point and directory structure to be owned by the user executing + // the application. This helps to prevent permissions issues. + string loggedInUser = this.PlatformSpecifics.GetLoggedInUser(); + + await this.systemManager.SetFullPermissionsAsync( + newMountPoint, + this.Platform, + telemetryContext, + CancellationToken.None, + loggedInUser, + this.Logger); + mountPointsCreated = true; } } @@ -190,4 +225,4 @@ private IEnumerable GetTargetDisks(IEnumerable disks, string diskFil return filteredDisks; } } -} +} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs b/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs index 4ee17e5f28..3c05a9033e 100644 --- a/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs +++ b/src/VirtualClient/VirtualClient.Dependencies/MySqlServer/MySqlServerConfiguration.cs @@ -302,7 +302,7 @@ private async Task GetMySQLInnodbDirectoriesAsync(CancellationToken canc foreach (Disk disk in disksToTest) { - diskPaths += $"{disk.GetPreferredAccessPath(this.Platform)};"; + diskPaths += $"{this.Combine(disk.GetPreferredAccessPath(this.Platform), "mysql")};"; } } diff --git a/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs b/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs index cd00a1affe..ddb880b7e1 100644 --- a/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs +++ b/src/VirtualClient/VirtualClient.Dependencies/PostgreSQLServer/PostgreSQLServerConfiguration.cs @@ -267,7 +267,7 @@ private async Task GetPostgreSQLInnodbDirectoriesAsync(CancellationToken foreach (Disk disk in disksToTest) { - diskPaths += $"{disk.GetPreferredAccessPath(this.Platform)};"; + diskPaths += $"{this.Combine(disk.GetPreferredAccessPath(this.Platform), "postgresql")};"; } } diff --git a/src/VirtualClient/VirtualClient.Main/CommandBase.cs b/src/VirtualClient/VirtualClient.Main/CommandBase.cs index b870ccae21..f664d77cbf 100644 --- a/src/VirtualClient/VirtualClient.Main/CommandBase.cs +++ b/src/VirtualClient/VirtualClient.Main/CommandBase.cs @@ -503,7 +503,7 @@ protected virtual IServiceCollection InitializeDependencies(string[] args) if (this.Verbose) { - this.LoggingLevel = LogLevel.Debug; + this.LoggingLevel = LogLevel.Trace; } else if (this.LoggingLevel == null) { diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json index 58c7745c19..43e7354597 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-OLTP.json @@ -161,7 +161,6 @@ "Type": "MountDisks", "Parameters": { "Scenario": "CreateMountPoints", - "MountLocation": "Root", "Role": "Server" } }, @@ -170,7 +169,7 @@ "Parameters": { "Scenario": "DownloadMySqlServerPackage", "BlobContainer": "packages", - "BlobName": "mysql-server-8.0.36-v2.zip", + "BlobName": "mysql.8.0.36.rev3.zip", "PackageName": "mysql-server", "Extract": true, "Role": "Server" diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-TPCC.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-TPCC.json index 02a8968e65..eb966e653c 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-TPCC.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-MYSQL-SYSBENCH-TPCC.json @@ -48,7 +48,6 @@ "Type": "MountDisks", "Parameters": { "Scenario": "CreateMountPoints", - "MountLocation": "Root", "Role": "Server" } }, @@ -57,7 +56,7 @@ "Parameters": { "Scenario": "DownloadMySqlServerPackage", "BlobContainer": "packages", - "BlobName": "mysql-server-8.0.36.zip", + "BlobName": "mysql.8.0.36.rev3.zip", "PackageName": "mysql-server", "Extract": true, "Role": "Server" diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCC.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCC.json index 616dc6d4a8..4f0378e86f 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCC.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-HAMMERDB-TPCC.json @@ -71,7 +71,7 @@ "Parameters": { "Scenario": "DownloadPostgreSQLPackage", "BlobContainer": "packages", - "BlobName": "postgresql.14.0.0.rev2.zip", + "BlobName": "postgresql.14.0.0.rev3.zip", "PackageName": "postgresql", "Extract": true } @@ -114,8 +114,8 @@ "Workload": "tpcc", "SQLServer": "postgresql", "PackageName": "hammerdb", - "VirtualUsers": "1", - "WarehouseCount": "1", + "VirtualUsers": 1, + "WarehouseCount": 1, "Port": "$.Parameters.Port", "Role": "Server" } diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-OLTP.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-OLTP.json index 54efc86deb..99073f5c4c 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-OLTP.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-OLTP.json @@ -162,7 +162,6 @@ "Type": "MountDisks", "Parameters": { "Scenario": "CreateMountPoints", - "MountLocation": "Root", "Role": "Server" } }, @@ -171,7 +170,7 @@ "Parameters": { "Scenario": "DownloadPostgreSQLServerPackage", "BlobContainer": "packages", - "BlobName": "postgresql.14.0.0.rev2.zip", + "BlobName": "postgresql.14.0.0.rev3.zip", "PackageName": "postgresql", "Extract": true, "Role": "Server" diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-TPCC.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-TPCC.json index 0c302682e6..95679d61b2 100644 --- a/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-TPCC.json +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-POSTGRESQL-SYSBENCH-TPCC.json @@ -50,7 +50,6 @@ "Type": "MountDisks", "Parameters": { "Scenario": "CreateMountPoints", - "MountLocation": "Root", "Role": "Server" } }, @@ -59,7 +58,7 @@ "Parameters": { "Scenario": "DownloadPostgreSQLServerPackage", "BlobContainer": "packages", - "BlobName": "postgresql.14.0.0.rev2.zip", + "BlobName": "postgresql.14.0.0.rev3.zip", "PackageName": "postgresql", "Extract": true, "Role": "Server" diff --git a/website/docs/guides/0220-usage-testing-disks.md b/website/docs/guides/0220-usage-testing-disks.md index a545ed0ca1..3d945a8eca 100644 --- a/website/docs/guides/0220-usage-testing-disks.md +++ b/website/docs/guides/0220-usage-testing-disks.md @@ -1,11 +1,40 @@ # Usage: Testing Disks VC automates disk formating and run storage workloads on the target systems. We documented the rules and the mechanism we used to identify disks to run workloads on. -## Rules -1. On OS Disk, only run on the OS partition. -2. If other disks have more than 1 partition, run on all concurrently that passes the filters (exclude /, /mnt, /boot.efi) -3. On Linux, only run on four prefix: `/dev/hd`, `/dev/sd`, `/dev/nvme`, `/dev/xvd`. -4. On Windows, disks with status set to offline and read-only will be filtered out. +## Disk Mount Points +On certain systems, Virtual Client may need to create mount points on each of the disks in order to have both permissions and a path to +operate disk I/O work. In general, Virtual Client must create mount points when the following is true: + +* Disks exist that do not have file system partitions/volumes (i.e. uninitialized disks). +* Disks exist with file system partitions/volumes but without any pre-existing mount points/access paths. + +The remainder of this section documents where disk mount points are created on the system when required. + +### Mount Points on Unix Systems +The following table illustrates the mount points that will be created when required on Unix systems. When logged in as the "root" on the system, +mount points will be created at the root of the file system directory structure. When logged in as a user other than "root" +(including when running as "sudo"), mount points will be created in the user home directory (e.g. /home/user) + +| Logged In User | Disk/Device Paths | Mount Points/Paths | +|----------------|-------------------|-------------------------| +| root | /dev/sdc1
/dev/sdd1 | /mnt_dev_sdc1
/mnt_dev_sdd1 | +| root | /dev/sdc1
/dev/sdc2
/dev/sdd1
/dev/sdd2 | /mnt_dev_sdc1
/mnt_dev_sdc2
/mnt_dev_sdd1
/mnt_dev_sdd2 | +| non-root user | /dev/sdc1
/dev/sdd1 | /mnt_dev_sdc1
/mnt_dev_sdd1 | +| non-root user | /dev/sdc1
/dev/sdc2
/dev/sdd1
/dev/sdd2 | /home/\{user\}/mnt_dev_sdc1
/home/\{user\}/mnt_dev_sdc2
/home/\{user\}/mnt_dev_sdd1
/home/\{user\}/mnt_dev_sdd2 | + +### Mount Points on Windows Systems +Mount points are NOT typically required on Windows systems once disks are formatted and have partitions/volumes. This is because Windows associates +a letter with each volume (e.g. C:\, D:\) and this the disk volume + file system is accessible through this path. + +Virtual Client does not currently support formatting disks beyond the first 26 disks due to inherent Windows limitations. Virtual Client can +operate on systems with more than 26 total partitions/volumes; however, these must be prepared before running the application + +## General Disk Testing Guidelines +1. When targeting the OS disk, ONLY the OS partition for that disk will be targeted. +2. For systems having disks with more than 1 partition, all partitions will be targeted. +3. On Unix systems having disks with more than 1 partition, the following device paths are excluded: ```/```, ```/mnt```, ```/boot/efi```. +3. On Unix systems, device paths having the following prefixes are targeted: `/dev/hd`, `/dev/sd`, `/dev/nvme`, `/dev/xvd`. +4. On Windows systems, disks that are offline or read-only will not be targeted. ## Disk Filters Different environments, Azure, AWS/GCP, Lab host, Lab VM, have vastly different configuration of disks. It is a challenge to have a schema that allows you to target the desired disk consistently. diff --git a/website/docs/workloads/diskspd/diskspd-profiles.md b/website/docs/workloads/diskspd/diskspd-profiles.md index ea799799e9..1949cb190d 100644 --- a/website/docs/workloads/diskspd/diskspd-profiles.md +++ b/website/docs/workloads/diskspd/diskspd-profiles.md @@ -2,7 +2,7 @@ The following profiles run customer-representative or benchmarking scenarios using the DiskSpd workload. * [Workload Details](./diskspd.md) -* [Testing Specific Disks](../../guides/0220-usage-testing-disks.md) +* [Testing Disks](../../guides/0220-usage-testing-disks.md) ## PERF-IO-DISKSPD.json Runs an high stress IO-intensive workload using the DiskSpd toolset to test performance of disks on the system. This profile is a Windows-only profile. @@ -51,7 +51,7 @@ aspects of the workload execution. * **Dependencies** The dependencies defined in the 'Dependencies' section of the profile itself are required in order to run the workload operations effectively. * Internet connection. - * Any 'DiskFilter' parameter value used should match the set of disks desired. See the link for 'Testing Specific Disks' above. + * Any 'DiskFilter' parameter value used should match the set of disks desired. See the link for 'Testing Disks' above. Additional information on components that exist within the 'Dependencies' section of the profile can be found in the following locations: * [Installing Dependencies](https://microsoft.github.io/VirtualClient/docs/category/dependencies/) @@ -89,7 +89,7 @@ aspects of the workload execution. | Parameter | Purpose | Default Value | |---------------------------|---------------------------------------------------------------------------------|---------------| - | DiskFilter | Optional. Filter allowing the user to select the disks on which to test.

See the link 'Testing Specific Disks' at the top for more details. | BiggestSize | + | DiskFilter | Optional. Filter allowing the user to select the disks on which to test.

See the link 'Testing Disks' at the top for more details. | BiggestSize | | DiskFillSize | Optional. Allows the user to override the default disk fill size used in the FIO profile (e.g. 500GB -> 26GB). This enables the profile to be used in scenarios where the disk size is very small (e.g. local/temp disk -> 32GB in size). | 500GB | | Duration | Optional. Defines the amount of time to run each FIO scenario/action within the profile. | 5 minutes | | FileSize | Optional. Allows the user to override the default file size used in the FIO profile (e.g. 496GB -> 26GB). This enables the profile to be used in scenarios where the disk size is very small (e.g. local/temp disk -> 32GB in size). | 496GB | @@ -105,7 +105,7 @@ aspects of the workload execution. | Scenario | Scenario use to define the given action of profile. This can be used to specify exact actions to run or exclude from the profile. | Any string | | MetricsScenario | The name to use as the "scenario" for all metrics output for the particular profile action. | | | CommandLine | The command line parameters for FIO tool set. | Any Valid FIO arguments | - | DiskFilter | Filter allowing the user to select the disks on which to test. | See the link 'Testing Specific Disks' at the top for more details. | + | DiskFilter | Filter allowing the user to select the disks on which to test. | See the link 'Testing Disks' at the top for more details. | | Duration | Defines the amount of time to run each FIO scenario/action within the profile. | integer or time span | | PackageName | The logical name for FIO package downloaded and that contains the toolset. | | | ProcessModel | Defines how the FIO processes will be executed. | SingleProcess
Executes a single FIO process running 1 job targeting I/O operations against each disk. Results are separated per-disk.

SingleProcessPerDisk
Executes a single FIO process for each disk with each process running 1 job targeting I/O operations against that disk (higher stress profile). Results are separated per-disk.

SingleProcessAggregated
Executes a single FIO process running 1 job per disk targeting I/O operations against that disk. Results are provided as an aggregation across all disks (i.e. a rollup). | @@ -119,7 +119,7 @@ aspects of the workload execution. number of system cores. * **Usage Examples** - The following section provides a few basic examples of how to use the workload profile. See the documentation at the top on 'Testing Specific Disks' + The following section provides a few basic examples of how to use the workload profile. See the documentation at the top on 'Testing Disks' for information on how to target select disks on the system. ``` bash diff --git a/website/docs/workloads/fio/fio-profiles.md b/website/docs/workloads/fio/fio-profiles.md index ec04be93cc..84bcb724dc 100644 --- a/website/docs/workloads/fio/fio-profiles.md +++ b/website/docs/workloads/fio/fio-profiles.md @@ -2,7 +2,7 @@ The following profiles run customer-representative or benchmarking scenarios using the Flexible I/O Tester (FIO) workload. * [Workload Details](./fio.md) -* [Testing Specific Disks](../../guides/0220-usage-testing-disks.md) +* [Testing Disks](../../guides/0220-usage-testing-disks.md) ## PERF-IO-FIO.json Runs an IO-intensive workload using the Flexible IO Tester (FIO) toolset to test performance of disks on the system. Although this profile @@ -53,7 +53,8 @@ aspects of the workload execution. * **Dependencies** The dependencies defined in the 'Dependencies' section of the profile itself are required in order to run the workload operations effectively. * Internet connection. - * Any 'DiskFilter' parameter value used should match the set of disks desired. See the link for 'Testing Specific Disks' above. + * Disk mount points exist for the disks to be targeted. Virtual Client will generally ensure that mount points exist by default. Details for mount point creation procedures can be found in the 'Testing Disks' documentation above. + * Any 'DiskFilter' parameter value used should match the set of disks desired. See the link for 'Testing Disks' above. Additional information on components that exist within the 'Dependencies' section of the profile can be found in the following locations: * [Installing Dependencies](https://microsoft.github.io/VirtualClient/docs/category/dependencies/) @@ -100,7 +101,7 @@ aspects of the workload execution. | Parameter | Purpose | Default Value | |---------------------------|---------------------------------------------------------------------------------|---------------| | DataIntegrityFileSize | Optional. Defines the size of the file/disk space that will be used for profile disk integrity scenarios/actions. | 4GB | - | DiskFilter | Optional. Filter allowing the user to select the disks on which to test.

See the link 'Testing Specific Disks' at the top for more details. | BiggestSize | + | DiskFilter | Optional. Filter allowing the user to select the disks on which to test.

See the link 'Testing Disks' at the top for more details. | BiggestSize | | DiskFillSize | Optional. Allows the user to override the default disk fill size used in the FIO profile (e.g. 500GB -> 26GB). This enables the profile to be used in scenarios where the disk size is very small (e.g. local/temp disk -> 32GB in size). | 500GB | | Duration | Optional. Defines the amount of time to run each FIO scenario/action within the profile. | 5 minutes | | Engine | Optional. Defines the I/O engine to use for the FIO operations (e.g. posixaio, libaio, windowsaio). | Linux = libaio, Windows = windowsaio | @@ -117,7 +118,7 @@ aspects of the workload execution. | Scenario | Scenario use to define the given action of profile. This can be used to specify exact actions to run or exclude from the profile. | Any string | | MetricsScenario | The name to use as the "scenario" for all metrics output for the particular profile action. | | | CommandLine | The command line parameters for FIO tool set. | Any Valid FIO arguments | - | DiskFilter | Filter allowing the user to select the disks on which to test. | See the link 'Testing Specific Disks' at the top for more details. | + | DiskFilter | Filter allowing the user to select the disks on which to test. | See the link 'Testing Disks' at the top for more details. | | Duration | Defines the amount of time to run each FIO scenario/action within the profile. | integer or time span | | Engine | Optional. Defines the I/O engine to use for the FIO operations (e.g. posixaio, libaio, windowsaio). | Linux = libaio, Windows = windowsaio | | PackageName | The logical name for FIO package downloaded and that contains the toolset. | | @@ -132,7 +133,7 @@ aspects of the workload execution. number of system cores. * **Usage Examples** - The following section provides a few basic examples of how to use the workload profile. See the documentation at the top on 'Testing Specific Disks' + The following section provides a few basic examples of how to use the workload profile. See the documentation at the top on 'Testing Disks' for information on how to target select disks on the system. ``` bash @@ -202,7 +203,8 @@ This profile uses an algorithm to determine the total number of jobs/threads as * **Dependencies** The dependencies defined in the 'Dependencies' section of the profile itself are required in order to run the workload operations effectively. * Internet connection. - * Any 'DiskFilter' parameter value used should match the set of disks desired. See the link for 'Testing Specific Disks' above. + * Disk mount points exist for the disks to be targeted. Virtual Client will generally ensure that mount points exist by default. Details for mount point creation procedures can be found in the 'Testing Disks' documentation above. + * Any 'DiskFilter' parameter value used should match the set of disks desired. See the link for 'Testing Disks' above. Additional information on components that exist within the 'Dependencies' section of the profile can be found in the following locations: * [Installing Dependencies](https://microsoft.github.io/VirtualClient/docs/category/dependencies/) @@ -376,7 +378,8 @@ This profile uses an algorithm to determine the amount of IOPS to run against th * **Dependencies** The dependencies defined in the 'Dependencies' section of the profile itself are required in order to run the workload operations effectively. * Internet connection. - * Any 'DiskFilter' parameter value used should match the set of disks desired. See the link for 'Testing Specific Disks' above. + * Disk mount points exist for the disks to be targeted. Virtual Client will generally ensure that mount points exist by default. Details for mount point creation procedures can be found in the 'Testing Disks' documentation above. + * Any 'DiskFilter' parameter value used should match the set of disks desired. See the link for 'Testing Disks' above. Additional information on components that exist within the 'Dependencies' section of the profile can be found in the following locations: * [Installing Dependencies](https://microsoft.github.io/VirtualClient/docs/category/dependencies/) @@ -507,7 +510,8 @@ Therefore, they are performed on different disks * **Dependencies** The dependencies defined in the 'Dependencies' section of the profile itself are required in order to run the workload operations effectively. * Internet connection. - * Any 'DiskFilter' parameter value used should match the set of disks desired. See the link for 'Testing Specific Disks' above. + * Disk mount points exist for the disks to be targeted. Virtual Client will generally ensure that mount points exist by default. Details for mount point creation procedures can be found in the 'Testing Disks' documentation above. + * Any 'DiskFilter' parameter value used should match the set of disks desired. See the link for 'Testing Disks' above. Additional information on components that exist within the 'Dependencies' section of the profile can be found in the following locations: * [Installing Dependencies](https://microsoft.github.io/VirtualClient/docs/category/dependencies/)