mysql minor changes and sysbench cleanup addition

src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/Sysbench/SysbenchExample1.txt

@@ -0,0 +1,38 @@
+sysbench 1.1.0 (using bundled LuaJIT 2.1.0-beta3)
+
+Running the test with following options:
+Number of threads: 8
+Initializing random number generator from current time
+
+
+Initializing worker threads...
+
+Threads started!
+
+SQL statistics:
+    queries performed:
+        read:                            9720106
+        write:                           0
+        other:                           0
+        total:                           9720106
+    transactions:                        9720106 (32400.24 per sec.)
+    queries:                             9720106 (32400.24 per sec.)
+    ignored errors:                      0      (0.00 per sec.)
+    reconnects:                          0      (0.00 per sec.)
+
+Throughput:
+    events/s (eps):                      32400.2399
+    time elapsed:                        300.0011s
+    total number of events:              9720106
+
+Latency (ms):
+         min:                                    0.16
+         avg:                                    0.25
+         max:                                   13.68
+         95th percentile:                        0.28
+         sum:                              2389444.97
+
+Threads fairness:
+    events (avg/stddev):           1215013.2500/22596.17
+    execution time (avg/stddev):   298.6806/0.05
+

src/VirtualClient/VirtualClient.Actions.UnitTests/Sysbench/SysbenchClientExecutorTests.cs

@@ -100,8 +100,6 @@ public async Task SysbenchClientExecutorRunsTheExpectedWorkloadCommand()
                     commandExecuted = true;
                 }
 
-                Assert.IsTrue(commandExecuted);
-
                 InMemoryProcess process = new InMemoryProcess
                 {
                     StartInfo = new ProcessStartInfo
@@ -124,6 +122,8 @@ public async Task SysbenchClientExecutorRunsTheExpectedWorkloadCommand()
             {
                 await SysbenchExecutor.ExecuteAsync(CancellationToken.None);
             }
+
+            Assert.IsTrue(commandExecuted);
         }
 
         [Test]
@@ -146,8 +146,6 @@ public async Task SysbenchClientExecutorUsesDefinedParametersWhenRunningTheWorkl
                     commandExecuted = true;
                 }
 
-                Assert.IsTrue(commandExecuted);
-
                 InMemoryProcess process = new InMemoryProcess
                 {
                     StartInfo = new ProcessStartInfo
@@ -170,6 +168,8 @@ public async Task SysbenchClientExecutorUsesDefinedParametersWhenRunningTheWorkl
             {
                 await SysbenchExecutor.ExecuteAsync(CancellationToken.None);
             }
+
+            Assert.IsTrue(commandExecuted);
         }
 
         [Test]
@@ -189,8 +189,6 @@ public async Task SysbenchClientExecutorRunsTheExpectedBalancedScenario()
                     commandExecuted = true;
                 }
 
-                Assert.IsTrue(commandExecuted);
-
                 InMemoryProcess process = new InMemoryProcess
                 {
                     StartInfo = new ProcessStartInfo
@@ -213,6 +211,8 @@ public async Task SysbenchClientExecutorRunsTheExpectedBalancedScenario()
             {
                 await SysbenchExecutor.ExecuteAsync(CancellationToken.None);
             }
+
+            Assert.IsTrue(commandExecuted);
         }
 
         [Test]
@@ -232,8 +232,6 @@ public async Task SysbenchClientExecutorRunsInMemoryScenario()
                     commandExecuted = true;
                 }
 
-                Assert.IsTrue(commandExecuted);
-
                 InMemoryProcess process = new InMemoryProcess
                 {
                     StartInfo = new ProcessStartInfo
@@ -256,6 +254,8 @@ public async Task SysbenchClientExecutorRunsInMemoryScenario()
             {
                 await SysbenchExecutor.ExecuteAsync(CancellationToken.None).ConfigureAwait(false);
             }
+
+            Assert.IsTrue(commandExecuted);
         }
 
         [Test]
@@ -275,8 +275,6 @@ public async Task SysbenchClientExecutorRunsTheExpectedTPCCWorkloadCommand()
                     commandExecuted = true;
                 }
 
-                Assert.IsTrue(commandExecuted);
-
                 InMemoryProcess process = new InMemoryProcess
                 {
                     StartInfo = new ProcessStartInfo
@@ -299,6 +297,8 @@ public async Task SysbenchClientExecutorRunsTheExpectedTPCCWorkloadCommand()
             {
                 await SysbenchExecutor.ExecuteAsync(CancellationToken.None);
             }
+
+            Assert.IsTrue(commandExecuted);
         }
 
         [Test]

src/VirtualClient/VirtualClient.Actions.UnitTests/Sysbench/SysbenchConfigurationTests.cs

@@ -77,7 +77,7 @@ public async Task SysbenchConfigurationSkipsSysbenchInitialization()
 
             string[] expectedCommands =
             {
-                $"python3 {this.mockPackagePath}/populate-database.py --dbName sbtest --databaseSystem MySQL --benchmark OLTP --tableCount 10 --recordCount 1000 --threadCount 8 --password [A-Za-z0-9+/=]+",
+                $"python3 {this.mockPackagePath}/populate-database.py --dbName sbtest --databaseSystem MySQL --benchmark OLTP --threadCount 8 --tableCount 10 --recordCount 1000 --password [A-Za-z0-9+/=]+",
             };
 
             int commandNumber = 0;
@@ -122,7 +122,7 @@ public async Task SysbenchConfigurationPreparesDatabase()
             string[] expectedCommands =
             {
                 $"python3 {this.mockPackagePath}/configure-workload-generator.py --distro Ubuntu --databaseSystem MySQL --packagePath {this.mockPackagePath}",
-                $"python3 {this.mockPackagePath}/populate-database.py --dbName sbtest --databaseSystem MySQL --benchmark OLTP --tableCount 10 --recordCount 1000 --threadCount 8 --password [A-Za-z0-9+/=]+",
+                $"python3 {this.mockPackagePath}/populate-database.py --dbName sbtest --databaseSystem MySQL --benchmark OLTP --threadCount 8 --tableCount 10 --recordCount 1000 --password [A-Za-z0-9+/=]+",
             };
 
             int commandNumber = 0;
@@ -173,7 +173,7 @@ public async Task SysbenchConfigurationUsesDefinedParametersWhenRunningTheWorklo
             string[] expectedCommands =
             {
                 $"python3 {this.mockPackagePath}/configure-workload-generator.py --distro Ubuntu --databaseSystem MySQL --packagePath {this.mockPackagePath}",
-                $"python3 {this.mockPackagePath}/populate-database.py --dbName sbtest --databaseSystem MySQL --benchmark OLTP --tableCount 40 --recordCount 1000 --threadCount 16 --password [A-Za-z0-9+/=]+",
+                $"python3 {this.mockPackagePath}/populate-database.py --dbName sbtest --databaseSystem MySQL --benchmark OLTP --threadCount 16 --tableCount 40 --recordCount 1000 --password [A-Za-z0-9+/=]+",
             };
 
             int commandNumber = 0;
@@ -386,7 +386,7 @@ public async Task SysbenchConfigurationProperlyExecutesPostgreSQLOLTPConfigurabl
 
             string[] expectedCommands =
             {
-                $"python3 {this.mockPackagePath}/populate-database.py --dbName sbtest --databaseSystem PostgreSQL --benchmark OLTP --tableCount 40 --recordCount 1000 --threadCount 16 --password [A-Za-z0-9+/=]+"
+                $"python3 {this.mockPackagePath}/populate-database.py --dbName sbtest --databaseSystem PostgreSQL --benchmark OLTP --threadCount 16 --tableCount 40 --recordCount 1000 --password [A-Za-z0-9+/=]+"
             };
 
             int commandNumber = 0;

src/VirtualClient/VirtualClient.Actions.UnitTests/Sysbench/SysbenchMetricsParserTests.cs

@@ -46,5 +46,17 @@ public void SysbenchParserParsesCorrectly()
             MetricAssert.Exists(metrics, "latency p95", 68.05, "milliseconds");
             MetricAssert.Exists(metrics, "latency sum", 7458385.25, "milliseconds");
         }
+
+        [Test]
+        public void SysbenchParserParsesMetricsMetadataCorrectly()
+        {
+            string rawText = File.ReadAllText(Path.Combine(examplesDirectory, "SysbenchExample1.txt"));
+            SysbenchMetricsParser parser = new SysbenchMetricsParser(rawText);
+
+            IList<Metric> metrics = parser.Parse();
+            Assert.AreEqual(17, metrics.Count);
+            Assert.IsTrue(metrics[0].Metadata.ContainsKey("sysbench_version"));
+            Assert.IsTrue(metrics[0].Metadata["sysbench_version"].Equals("1.1.0"));
+        }
     }
 }

src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchClientExecutor.cs

@@ -6,7 +6,7 @@ namespace VirtualClient.Actions
     using System;
     using System.Collections.Generic;
     using System.Linq;
-    using System.Net;
+    using System.Text.RegularExpressions;
     using System.Threading;
     using System.Threading.Tasks;
     using Microsoft.Extensions.DependencyInjection;
@@ -137,9 +137,23 @@ private void CaptureMetrics(IProcessProxy process, EventContext telemetryContext
                 if (!string.IsNullOrEmpty(text)) 
                 {
                     try
-                    {
+                    {                        
                         SysbenchMetricsParser parser = new SysbenchMetricsParser(text);
                         IList<Metric> metrics = parser.Parse();
+                        string sysbenchVersion = null;
+
+                        var sysbenchVersionMetric = metrics.FirstOrDefault();
+                        if (sysbenchVersionMetric != null && sysbenchVersionMetric.Metadata.TryGetValue("sysbench_version", out var versionValue))
+                        {
+                            sysbenchVersion = versionValue?.ToString();
+                        }
+
+                        if (!string.IsNullOrEmpty(sysbenchVersion))
+                        {
+                            this.MetadataContract.Add("sysbench_version", sysbenchVersion, MetadataContractCategory.Dependencies);
+                        }
+
+                        this.MetadataContract.Apply(telemetryContext);
 
                         this.Logger.LogMetrics(
                             toolName: "Sysbench",
@@ -150,7 +164,8 @@ private void CaptureMetrics(IProcessProxy process, EventContext telemetryContext
                             null,
                             scenarioArguments: this.sysbenchLoggingArguments,
                             this.Tags,
-                            telemetryContext);
+                            telemetryContext,
+                            toolVersion: sysbenchVersion);
                     }
                     catch (Exception exc)
                     {
@@ -166,6 +181,14 @@ private Task ExecuteWorkloadAsync(EventContext telemetryContext, CancellationTok
             {
                 using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken))
                 {
+                    if (this.DatabaseSystem == "MySQL")
+                    {
+                        string mysqlVersion = await this.GetMySQLVersionAsync(telemetryContext, cancellationToken);
+
+                        this.MetadataContract.Add("mysql_version", mysqlVersion, MetadataContractCategory.Dependencies);
+                        this.MetadataContract.Apply(telemetryContext);
+                    }
+
                     if (this.Benchmark == BenchmarkName.OLTP)
                     {
                         if (this.Action == ClientAction.TruncateDatabase)
@@ -182,6 +205,10 @@ private Task ExecuteWorkloadAsync(EventContext telemetryContext, CancellationTok
                         {
                             await this.PrepareOLTPMySQLDatabase(telemetryContext, cancellationToken);
                         }
+                        else if (this.Action == ClientAction.Cleanup)
+                        {
+                            await this.CleanUpDatabase(telemetryContext, cancellationToken);
+                        }
                         else
                         {
                             await this.RunOLTPWorkloadAsync(telemetryContext, cancellationToken);
@@ -229,6 +256,33 @@ private async Task RunOLTPWorkloadAsync(EventContext telemetryContext, Cancellat
             }
         }
 
+        private async Task CleanUpDatabase(EventContext telemetryContext, CancellationToken cancellationToken)
+        {
+            int tableCount = GetTableCount(this.DatabaseScenario, this.TableCount, this.Workload);
+
+            string serverIp = (this.GetLayoutClientInstances(ClientRole.Server, false) ?? Enumerable.Empty<ClientInstance>())
+                                    .FirstOrDefault()?.IPAddress
+                                    ?? "localhost";
+
+            string sysbenchCleanupArguments = $"--dbName {this.DatabaseName} --databaseSystem {this.DatabaseSystem} --benchmark {this.Benchmark} --tableCount {tableCount}  --hostIpAddress {serverIp}";
+
+            string script = $"{this.SysbenchPackagePath}/cleanup-database.py ";
+
+            using (IProcessProxy process = await this.ExecuteCommandAsync(
+                SysbenchExecutor.PythonCommand,
+                script + sysbenchCleanupArguments,
+                this.SysbenchPackagePath,
+                telemetryContext,
+                cancellationToken))
+            {
+                if (!cancellationToken.IsCancellationRequested)
+                {
+                    await this.LogProcessDetailsAsync(process, telemetryContext, "Sysbench", logToFile: true);
+                    process.ThrowIfErrored<WorkloadException>(process.StandardError.ToString(), ErrorReason.WorkloadFailed);
+                }
+            }
+        }
+
         private async Task RunTPCCWorkloadAsync(EventContext telemetryContext, CancellationToken cancellationToken)
         {
             int tableCount = GetTableCount(this.Scenario, this.TableCount, this.Workload);
@@ -288,7 +342,7 @@ private async Task PrepareOLTPMySQLDatabase(EventContext telemetryContext, Cance
             int threadCount = GetThreadCount(this.SystemManager, this.DatabaseScenario, this.Threads);
             int recordCount = GetRecordCount(this.SystemManager, this.DatabaseScenario, this.RecordCount);
 
-            this.sysbenchLoggingArguments = $"--dbName {this.DatabaseName} --databaseSystem {this.DatabaseSystem} --benchmark {this.Benchmark} --tableCount {tableCount} --recordCount {recordCount} --threadCount {threadCount}";
+            this.sysbenchLoggingArguments = $"--dbName {this.DatabaseName} --databaseSystem {this.DatabaseSystem} --benchmark {this.Benchmark} --threadCount {threadCount} --tableCount {tableCount} --recordCount {recordCount}";
             this.sysbenchPrepareArguments = $"{this.sysbenchLoggingArguments} --password {this.SuperUserPassword}";
 
             string serverIp = (this.GetLayoutClientInstances(ClientRole.Server, false) ?? Enumerable.Empty<ClientInstance>())
@@ -315,5 +369,28 @@ private async Task PrepareOLTPMySQLDatabase(EventContext telemetryContext, Cance
                 }
             }
         }
+
+        /// <summary>
+        /// Returns MySQL Version.
+        /// </summary>
+        /// <returns></returns>
+        /// <exception cref="Exception"></exception>
+        private async Task<string> GetMySQLVersionAsync(EventContext telemetryContext, CancellationToken cancellationToken)
+        {
+            try
+            { 
+                IProcessProxy mysqlversionprocess = await this.ExecuteCommandAsync("sudo", $"mysql -u {this.DatabaseName} -h {this.ServerIpAddress} -e \"SELECT VERSION();\"", Environment.CurrentDirectory, telemetryContext, cancellationToken);
+                string mysqlVersion = mysqlversionprocess.StandardOutput.ToString();
+
+                Regex regex = new Regex(@"(\d+\.\d+\.\d+)");
+                Match match = regex.Match(mysqlVersion);
+
+                return match.Success ? match.Groups[1].Value : string.Empty;
+            }
+            catch (Exception)
+            {
+                return string.Empty;
+            }
+        }
     }
 }

src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchConfiguration.cs

@@ -82,7 +82,7 @@ private async Task PrepareOLTPMySQLDatabase(EventContext telemetryContext, Cance
             int threadCount = GetThreadCount(this.SystemManager, this.DatabaseScenario, this.Threads);
             int recordCount = GetRecordCount(this.SystemManager, this.DatabaseScenario, this.RecordCount);
 
-            string sysbenchLoggingArguments = $"--dbName {this.DatabaseName} --databaseSystem {this.DatabaseSystem} --benchmark {this.Benchmark} --tableCount {tableCount} --recordCount {recordCount} --threadCount {threadCount}";
+            string sysbenchLoggingArguments = $"--dbName {this.DatabaseName} --databaseSystem {this.DatabaseSystem} --benchmark {this.Benchmark} --threadCount {threadCount} --tableCount {tableCount} --recordCount {recordCount}";
             this.sysbenchPrepareArguments = $"{sysbenchLoggingArguments} --password {this.SuperUserPassword}";
 
             string serverIp = "localhost";

src/VirtualClient/VirtualClient.Actions/Sysbench/SysbenchExecutor.cs

@@ -135,7 +135,7 @@ public int? Threads
         }
 
         /// <summary>
-        /// Number of records per table.
+        /// Database system used. e.g: MYSQL, PostgreSQL.
         /// </summary>
         public string DatabaseSystem
         {
@@ -417,7 +417,7 @@ protected void AddMetric(string arguments, IProcessProxy process, EventContext t
 
                 List<Metric> metrics = new List<Metric>();
                 double duration = (process.ExitTime - process.StartTime).TotalMinutes;
-                metrics.Add(new Metric("PopulateDatabaseTime_Minutes ", duration, "minutes", MetricRelativity.LowerIsBetter));
+                metrics.Add(new Metric("PopulateDatabaseDuration", duration, "minutes", MetricRelativity.LowerIsBetter));
 
                 this.Logger.LogMetrics(
                     toolName: "Sysbench",
@@ -532,9 +532,14 @@ internal class ClientAction
             public const string TruncateDatabase = nameof(TruncateDatabase);
 
             /// <summary>
-            /// Truncates all tables existing in database
+            /// Runs specified workload.
             /// </summary>
             public const string RunWorkload = nameof(RunWorkload);
+
+            /// <summary>
+            /// Runs sysbench cleanup action.
+            /// </summary>
+            public const string Cleanup = nameof(Cleanup);
         }
     }
 }