Added the ability to set a random seed to replicate test runs

CHANGELIST.md

@@ -8,6 +8,7 @@
   - Added a Parameter thread safety test which attempts to call setValue on parameters from multiple threads concurrently mimicking automation and GUI interaction
   - Added a PluginStateTestRestoration test which calculates some checksums before randomising and restoring plugin state to check state restoration is correct
   - Added the ability to set command line arguments as environment variables e.g. "--skip-gui-tests" > "SKIP_GUI_TESTS=1", "--timeout-ms 30000" > "TIMEOUT_MS=30000"
+  - Added the ability to set a random seed to use for the tests
 
 ### 0.1.4
   - Added stack backtraces to crashed validation output

Source/CommandLine.cpp

@@ -172,6 +172,31 @@ static int getStrictnessLevel (const StringArray& args)
     return 5;
 }
 
+int64 getRandomSeed (const StringArray& args)
+{
+    const int seedIndex = indexOfArgument (args, "random-seed");
+
+    if (seedIndex != -1)
+    {
+        if (args.size() > seedIndex)
+        {
+            auto seedString = args[seedIndex + 1];
+
+            if (! seedString.containsOnly ("x-0123456789"))
+                throw CommandLineError ("Invalid random seed argument!");
+
+            if (seedString.startsWith ("0x"))
+                return seedString.getHexValue64();
+
+            return seedString.getLargeIntValue();
+        }
+
+        throw CommandLineError ("Missing random seed argument!");
+    }
+
+    return 0;
+}
+
 int64 getTimeout (const StringArray& args)
 {
     const int timeoutIndex = indexOfArgument (args, "timeout-ms");
@@ -232,6 +257,7 @@ static void validate (CommandLineValidator& validator, const StringArray& args)
             const bool validateInProcess = containsArgument (args, "validate-in-process");
             PluginTests::Options options;
             options.strictnessLevel = getStrictnessLevel (args);
+            options.randomSeed = getRandomSeed (args);
             options.timeoutMs = getTimeout (args);
             options.verbose = containsArgument (args, "verbose");
             options.dataFile = getDataFile (args);
@@ -327,6 +353,8 @@ static void showHelp()
               << "    Validates the files (or IDs for AUs)." << std::endl
               << "  --strictness-level [1-10]" << std::endl
               << "    Sets the strictness level to use. A minimum level of 5 (also the default) is recomended for compatibility. Higher levels include longer, more thorough tests such as fuzzing." << std::endl
+              << "  --random-seed [hex or int]" << std::endl
+              << "    Sets the random seed to use for the tests. Useful for replicating test environments." << std::endl
               << "  --timeout-ms [numMilliseconds]" << std::endl
               << "    Sets a timout which will stop validation with an error if no output from any test has happened for this number of ms." << std::endl
               << "    By default this is 30s but can be set to -1 to never timeout." << std::endl

Source/MainComponent.cpp

@@ -26,6 +26,16 @@ int getStrictnessLevel()
     return jlimit (1, 10, getAppPreferences().getIntValue ("strictnessLevel", 5));
 }
 
+void setRandomSeed (int64 newSeed)
+{
+    getAppPreferences().setValue ("randomSeed", newSeed);
+}
+
+int64 getRandomSeed()
+{
+    return getAppPreferences().getIntValue ("randomSeed", 0);
+}
+
 void setValidateInProcess (bool shouldValidateInProcess)
 {
     getAppPreferences().setValue ("validateInProcess", shouldValidateInProcess);
@@ -60,13 +70,35 @@ PluginTests::Options getTestOptions()
 {
     PluginTests::Options options;
     options.strictnessLevel = getStrictnessLevel();
+    options.randomSeed = getRandomSeed();
     options.timeoutMs = getTimeoutMs();
     options.verbose = getVerboseLogging();
 
     return options;
 }
 
 //==============================================================================
+void showRandomSeedDialog()
+{
+    const String message = TRANS("Set the random seed to use for the tests, useful for replicating issues");
+    std::shared_ptr<AlertWindow> aw (LookAndFeel::getDefaultLookAndFeel().createAlertWindow (TRANS("Set Random Seed"), message,
+                                                                                             TRANS("OK"), TRANS("Cancel"), String(),
+                                                                                             AlertWindow::QuestionIcon, 2, nullptr));
+    aw->addTextEditor ("randomSeed", String (getRandomSeed()));
+    aw->enterModalState (true, ModalCallbackFunction::create ([aw] (int res)
+                                                              {
+                                                                  if (res == 1)
+                                                                  {
+                                                                      if (auto te = aw->getTextEditor ("randomSeed"))
+                                                                      {
+                                                                          auto seedString = te->getText();
+                                                                          setRandomSeed (seedString.startsWith ("0x") ? seedString.getHexValue64()
+                                                                                                                      : seedString.getLargeIntValue());
+                                                                      }
+                                                                  }
+                                                              }));
+}
+
 void showTimeoutDialog()
 {
     const String message = TRANS("Set the duration in milliseconds after which to kill the validation if there has been no output from it");
@@ -159,13 +191,15 @@ MainComponent::MainComponent (Validator& v)
             enum MenuItem
             {
                 validateInProcess = 1,
+                showRandomSeed,
                 showTimeout,
                 verboseLogging,
                 showSettingsDir
             };
 
             PopupMenu m;
             m.addItem (validateInProcess, TRANS("Validate in process"), true, getValidateInProcess());
+            m.addItem (showRandomSeed, TRANS("Set random seed (123)").replace ("123", "0x" + String::toHexString (getRandomSeed()) + "/" + String (getRandomSeed())));
             m.addItem (showTimeout, TRANS("Set timeout (123ms)").replace ("123", String (getTimeoutMs())));
             m.addItem (verboseLogging, TRANS("Verbose logging"), true, getVerboseLogging());
             m.addSeparator();
@@ -178,6 +212,10 @@ MainComponent::MainComponent (Validator& v)
                                      setValidateInProcess (! getValidateInProcess());
                                      sp->validator.setValidateInProcess (getValidateInProcess());
                                  }
+                                 else if (res == showRandomSeed)
+                                 {
+                                     showRandomSeedDialog();
+                                 }
                                  else if (res == showTimeout)
                                  {
                                      showTimeoutDialog();

Source/PluginTests.cpp

@@ -103,7 +103,7 @@ std::unique_ptr<AudioPluginInstance> PluginTests::testOpenPlugin (const PluginDe
 void PluginTests::testType (const PluginDescription& pd)
 {
     logMessage ("\nTesting plugin: " + pd.createIdentifierString());
-    logMessage (pd.manufacturerName + "," + pd.name + ", " + pd.version);
+    logMessage (pd.manufacturerName + ": " + pd.name + " v" + pd.version);
 
     {
         beginTest ("Open plugin (cold)");

Source/PluginTests.h

@@ -27,6 +27,7 @@ struct PluginTests : public UnitTest
     struct Options
     {
         int strictnessLevel = 5;    /**< Max test level to run. */
+        int64 randomSeed = 0;       /**< The seed to use for the tests, 0 signifies a randomly generated seed. */
         int64 timeoutMs = 30000;    /**< Timeout after which to kill the test. */
         bool verbose = false;       /**< Whether or not to log additional information. */
         bool withGUI = true;        /**< Whether or not avoid tests that instantiate a gui. */

Source/Validator.cpp

@@ -103,7 +103,7 @@ inline Array<UnitTestRunner::TestResult> runTests (PluginTests& test, std::funct
 
     Array<UnitTest*> testsToRun;
     testsToRun.add (&test);
-    testRunner.runTests (testsToRun);
+    testRunner.runTests (testsToRun, options.randomSeed);
 
     for (int i = 0; i < testRunner.getNumResults(); ++i)
         results.add (*testRunner.getResult (i));
@@ -139,6 +139,7 @@ namespace IDs
     DECLARE_ID(fileOrID)
     DECLARE_ID(pluginDescription)
     DECLARE_ID(strictnessLevel)
+    DECLARE_ID(randomSeed)
     DECLARE_ID(timeoutMs)
     DECLARE_ID(verbose)
     DECLARE_ID(dataFile)
@@ -360,6 +361,7 @@ class ValidatorSlaveProcess : public ChildProcessSlave,
         {
             PluginTests::Options options;
             options.strictnessLevel = v.getProperty (IDs::strictnessLevel, 5);
+            options.randomSeed = v[IDs::randomSeed];
             options.timeoutMs = v.getProperty (IDs::timeoutMs, -1);
             options.verbose = v.getProperty (IDs::verbose, false);
             options.dataFile = File (v.getProperty (IDs::dataFile, String()));
@@ -546,6 +548,7 @@ class ValidatorMasterProcess    : public ChildProcessMaster
     {
         ValueTree v (IDs::PLUGINS);
         v.setProperty (IDs::strictnessLevel, options.strictnessLevel, nullptr);
+        v.setProperty (IDs::randomSeed, options.randomSeed, nullptr);
         v.setProperty (IDs::timeoutMs, options.timeoutMs, nullptr);
         v.setProperty (IDs::verbose, options.verbose, nullptr);
         v.setProperty (IDs::dataFile, options.dataFile.getFullPathName(), nullptr);