Merge pull request #168 from VJalili/benchmark

Improvements to the benchmarking scripts
Genometric · Sep 5, 2022 · a6a5c53 · a6a5c53
2 parents f1a94e5 + bb90a8d
commit a6a5c53
Show file tree

Hide file tree

Showing 4 changed files with 25 additions and 6 deletions.
diff --git a/Benchmark/PerformanceTest.cs b/Benchmark/PerformanceTest.cs
@@ -29,15 +29,16 @@ public static void Test(string dataDir, string resultsFilename, string version,
                 var msg = $"\t[{++counter}/{cases.Count}]\tBenchmarking using {c.Key}: ... ";
                 Console.Write(msg);
                 var reps = SyntheticReps.Generate(c.Value, maxRepCount);
+                var syntheticReps = reps.Except(c.Value).ToList();
 
                 var minRepCount = 2;
                 for (int i = minRepCount; i <= maxRepCount; i++)
                 {
                     var testReps = reps.Take(i).ToList();
-
                     verInfo.InputFiles = testReps;
                     var result = MeasurePerformance(verInfo.StartInfo);
                     result.Version = verInfo.Version;
+                    result.ExperimentId = c.Key;
                     result.ReplicateCount = testReps.Count;
                     foreach (var filename in testReps)
                         result.IntervalCount += GetPeaksCount(filename);
@@ -47,6 +48,10 @@ public static void Test(string dataDir, string resultsFilename, string version,
                     Console.Write($"\r{msg}{Math.Floor((i - minRepCount) / (double)(maxRepCount - minRepCount) * 100)}%");
                 }
 
+                foreach (var syntheticRep in syntheticReps)
+                    File.Delete(syntheticRep);
+                if (verInfo.OutputDir != null)
+                    Directory.Delete(verInfo.OutputDir, true);
                 timer.Stop();
                 Console.WriteLine($"\r{msg}Done!\t(ET: {timer.Elapsed}");
 
@@ -57,7 +62,7 @@ public static void Test(string dataDir, string resultsFilename, string version,
             }
         }
 
-        private static Result MeasurePerformance(ProcessStartInfo info, int waitMilliseconds = 100)
+        private static Result MeasurePerformance(ProcessStartInfo info, int waitToExitInMilliseconds = 100)
         {
             var result = new Result();
 
@@ -92,7 +97,7 @@ private static Result MeasurePerformance(ProcessStartInfo info, int waitMillisec
                         process.StandardOutput.ReadToEnd();
                     }
                 }
-                while (!process.WaitForExit(waitMilliseconds));
+                while (!process.WaitForExit(waitToExitInMilliseconds));
                 result.Runtime.Stop();
             }
 

diff --git a/Benchmark/Result.cs b/Benchmark/Result.cs
@@ -8,6 +8,8 @@ public class Result
 
         public string Version { set; get; } = "not_specified";
 
+        public string ExperimentId { set; get; } = "not_specified";
+
         public int ReplicateCount { set; get; }
 
         public int IntervalCount { set; get; }
@@ -35,6 +37,7 @@ public static string GetHeader(string delimiter = _delimiter)
             return string.Join(delimiter, new string[]
             {
                 "mspc_version",
+                "experiment_id",
                 "replicate_count",
                 "interval_count",
                 "runtime_seconds",
@@ -49,6 +52,7 @@ public string ToString(string delimiter = _delimiter)
             return string.Join(delimiter, new string[]
             {
                 Version,
+                ExperimentId,
                 ReplicateCount.ToString(),
                 IntervalCount.ToString(),
                 Runtime.Elapsed.TotalSeconds.ToString(),

diff --git a/Benchmark/SyntheticReps.cs b/Benchmark/SyntheticReps.cs
@@ -13,14 +13,14 @@ public static List<string> Generate(List<string> replicates, int maxRepCount)
             {
                 extReps.Add(RandomAlter(
                     filename: replicates[i],
-                    filenamePostfix: "_rnd_" + (extReps.Count + 1).ToString(),
+                    filenamePostfix: "_synthetic_" + (extReps.Count + 1).ToString(),
                     rndSeed: (extReps.Count * 10) + replicates[i].Length));
 
                 if (++i >= replicates.Count)
                     i = 0;
             }
 
-            return replicates;
+            return extReps;
         }
 
         public static string RandomAlter(string filename, string filenamePostfix, int rndSeed = 0)

diff --git a/Benchmark/VersionInfo.cs b/Benchmark/VersionInfo.cs
@@ -26,6 +26,8 @@ public ProcessStartInfo StartInfo
 
         public Uri ReleaseUri { private set; get; }
 
+        public string? OutputDir { private set; get; } = null;
+
 
         public VersionInfo(
             string version,
@@ -49,11 +51,12 @@ private bool TryRunVerSpecificConfig(string version)
             {
                 _invocation = "mspc.exe";
                 ReleaseUri = new Uri(ReleaseUri, $"download/{version}/mspc.zip");
+                SetOutputDir();
                 return true;
             }
 
             pattern = new Regex(@"^v2\.\d+(\.\d+)?$");
-            if(pattern.IsMatch(version))
+            if (pattern.IsMatch(version))
             {
                 _invocation = "mspc.exe";
                 ReleaseUri = new Uri(ReleaseUri, $"download/{version}/v2.1.zip");
@@ -83,5 +86,12 @@ private async Task<(bool, string)> TryLocalize()
 
             return (true, dir);
         }
+
+        private void SetOutputDir()
+        {
+            OutputDir = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
+            Directory.CreateDirectory(OutputDir);
+            Args += $" -o {Path.Combine(OutputDir, "tmp")}";
+        }
     }
 }