From b8754ab7a1f88153709b152fe4ff2d702dd9a1cd Mon Sep 17 00:00:00 2001 From: David Rowland Date: Sat, 6 Jun 2026 21:58:39 +0100 Subject: [PATCH 01/12] Bump pluginval to C++23 and add magic_args + nlohmann/json via CPM Prepares for the CLI parser refactor onto magic_args + a JSON-merge settings pipeline. - target_compile_features: cxx_std_20 -> cxx_std_23 (magic_args requires C++23) - CPMAddPackage magic_args v0.2.1 (header-only INTERFACE: magic_args::magic_args) - CPMAddPackage nlohmann/json 3.12.0 (nlohmann_json::nlohmann_json) - link both into the pluginval target Verified: clean build + link under C++23 with juce_recommended_warning_flags on JUCE 8.0.13 (macOS/AppleClang); binary runs. Co-Authored-By: Claude Opus 4.8 --- CMakeLists.txt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f2ac5bb..3b314a1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,6 +61,8 @@ option(JUCE_ENABLE_MODULE_SOURCE_GROUPS "Enable Module Source Groups" ON) include(cmake/CPM.cmake) CPMAddPackage("gh:Neargye/magic_enum#v0.9.7") +CPMAddPackage("gh:fredemmott/magic_args#v0.2.1") +CPMAddPackage("gh:nlohmann/json@3.12.0") if(PLUGINVAL_ENABLE_RTCHECK) CPMAddPackage("gh:Tracktion/rtcheck#main") @@ -109,7 +111,7 @@ juce_add_gui_app(pluginval juce_generate_juce_header(pluginval) -target_compile_features(pluginval PRIVATE cxx_std_20) +target_compile_features(pluginval PRIVATE cxx_std_23) set_target_properties(pluginval PROPERTIES C_VISIBILITY_PRESET hidden @@ -176,7 +178,9 @@ target_link_libraries(pluginval PRIVATE juce::juce_audio_processors juce::juce_audio_utils juce::juce_recommended_warning_flags - magic_enum) + magic_enum + magic_args::magic_args + nlohmann_json::nlohmann_json) if (${CMAKE_SYSTEM_NAME} MATCHES "Linux") target_link_libraries(pluginval PRIVATE From ed9cba259a0553d84351f9dbbf4cc5536588221a Mon Sep 17 00:00:00 2001 From: David Rowland Date: Sat, 6 Jun 2026 22:20:45 +0100 Subject: [PATCH 02/12] Replace hand-rolled CLI parser with a JSON-merge settings pipeline Introduces a single parser-agnostic settings struct and a layered JSON pipeline that replaces the bespoke juce::ArgumentList parser. Pipeline: each input layer (--config file, environment, CLI) becomes a sparse nlohmann::json object; layers are merged with merge_patch in precedence order (defaults < config < env < CLI, CLI wins) and deserialised in one step into PluginvalSettings, which converts to the existing PluginTests::Options at the JUCE boundary. - PluginvalSettings.h: unified std-typed struct + toPluginTestOptions() + fromPluginTestOptions(); NLOHMANN_DEFINE_TYPE..._WITH_DEFAULT so missing keys fall back to defaults; RealtimeCheck enum<->string mapping. - SettingsSerializer.{h,cpp}: JSON load/save + the comma-list, hex-seed and disabled-tests(file-or-list) coercions shared by the CLI/env layers. - SettingsParser.{h,cpp}: preprocess (deprecation rewrite, macOS flag strip, implicit --validate), magic_args parse of the flat options into the sparse CLI layer, env layer, merge, and the child-process handoff. - Child process round-trip now uses an authoritative base64 JSON arg (--config-base64) instead of per-flag re-serialisation, avoiding quote/tokenisation hazards. Round-trip asserted in tests. - CommandLine.{h,cpp}: thin adapter; old manual parsers/env-merge/help text deleted. magic_args drives --help (auto usage + env-var trailer). - New --config option; precedence covered by tests. - CommandLineTests.cpp rewritten against the new pipeline (defaults, parser, hex seed, comma lists, rtcheck, path resolution, implicit validate, env precedence, config precedence, round-trip). - CMake: macOS deployment target 10.11 -> 13.3 (std::format in magic_args requires libc++ floating-point to_chars, available on 13.3+); add new sources. All --run-tests pass. Co-Authored-By: Claude Opus 4.8 --- CMakeLists.txt | 10 +- Source/CommandLine.cpp | 533 ++++------------------------------ Source/CommandLine.h | 4 +- Source/CommandLineTests.cpp | 271 ++++++++++++----- Source/PluginvalSettings.h | 142 +++++++++ Source/SettingsParser.cpp | 363 +++++++++++++++++++++++ Source/SettingsParser.h | 72 +++++ Source/SettingsSerializer.cpp | 88 ++++++ Source/SettingsSerializer.h | 56 ++++ 9 files changed, 983 insertions(+), 556 deletions(-) create mode 100644 Source/PluginvalSettings.h create mode 100644 Source/SettingsParser.cpp create mode 100644 Source/SettingsParser.h create mode 100644 Source/SettingsSerializer.cpp create mode 100644 Source/SettingsSerializer.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 3b314a1..932c81a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,8 +6,9 @@ project(pluginval VERSION ${CURRENT_VERSION}) # Just compliing pluginval if (pluginval_IS_TOP_LEVEL) if (APPLE) - # Target OS versions down to 10.11 - set (CMAKE_OSX_DEPLOYMENT_TARGET "10.11" CACHE INTERNAL "") + # Target OS versions down to 13.3. std::format (used by magic_args) pulls in + # libc++'s floating-point formatter, whose to_chars is only available on 13.3+. + set (CMAKE_OSX_DEPLOYMENT_TARGET "13.3" CACHE INTERNAL "") # Uncomment to produce a universal binary # set(CMAKE_OSX_ARCHITECTURES arm64 x86_64) @@ -122,6 +123,9 @@ set(SourceFiles Source/CrashHandler.h Source/MainComponent.h Source/PluginTests.h + Source/PluginvalSettings.h + Source/SettingsParser.h + Source/SettingsSerializer.h Source/TestUtilities.h Source/Validator.h Source/CommandLine.cpp @@ -129,6 +133,8 @@ set(SourceFiles Source/Main.cpp Source/MainComponent.cpp Source/PluginTests.cpp + Source/SettingsParser.cpp + Source/SettingsSerializer.cpp Source/tests/BasicTests.cpp Source/tests/LocaleTest.cpp Source/tests/BusTests.cpp diff --git a/Source/CommandLine.cpp b/Source/CommandLine.cpp index f5cd703..900bad0 100644 --- a/Source/CommandLine.cpp +++ b/Source/CommandLine.cpp @@ -16,6 +16,11 @@ #include "Validator.h" #include "CrashHandler.h" #include "PluginTests.h" +#include "PluginvalSettings.h" +#include "SettingsParser.h" + +#include +#include #if JUCE_MAC #include @@ -23,8 +28,6 @@ #include #endif -#include - //============================================================================== static void exitWithError (const juce::String& error) { @@ -115,305 +118,7 @@ void CommandLineValidator::validate (const juce::String& fileOrID, PluginTests:: }); } - -//============================================================================== -//============================================================================== -namespace -{ - juce::ArgumentList::Argument getArgumentAfterOption (const juce::ArgumentList& args, juce::StringRef option) - { - for (int i = 0; i < args.size() - 1; ++i) - if (args[i] == option) - return args[i + 1]; - - return {}; - } - - juce::var getOptionValue (const juce::ArgumentList& args, juce::StringRef option, juce::var defaultValue, juce::StringRef errorMessage) - { - if (args.containsOption (option)) - { - const auto nextArg = getArgumentAfterOption (args, option); - - if (nextArg.isShortOption() || nextArg.isLongOption()) - juce::ConsoleApplication::fail (errorMessage, -1); - - return nextArg.text; - } - - return defaultValue; - } - - int getStrictnessLevel (const juce::ArgumentList& args) - { - return juce::jlimit (1, 10, (int) getOptionValue (args, "--strictness-level", 5, "Missing strictness level argument! (Must be between 1 - 10)")); - } - - juce::int64 getRandomSeed (const juce::ArgumentList& args) - { - const juce::String seedString = getOptionValue (args, "--random-seed", "0", "Missing random seed argument!").toString(); - - if (! seedString.containsOnly ("x-0123456789acbdef")) - juce::ConsoleApplication::fail ("Invalid random seed argument!", -1); - - if (seedString.startsWith ("0x")) - return seedString.getHexValue64(); - - return seedString.getLargeIntValue(); - } - - juce::int64 getTimeout (const juce::ArgumentList& args) - { - return getOptionValue (args, "--timeout-ms", 30000, "Missing timeout-ms level argument!"); - } - - int getNumRepeats (const juce::ArgumentList& args) - { - return juce::jmax (1, (int) getOptionValue (args, "--repeat", 1, "Missing repeat argument! (Must be greater than 0)")); - } - - juce::File getDataFile (const juce::ArgumentList& args) - { - return getOptionValue (args, "--data-file", {}, "Missing data-file path argument!").toString(); - } - - juce::File getOutputDir (const juce::ArgumentList& args) - { - return getOptionValue (args, "--output-dir", {}, "Missing output-dir path argument!").toString(); - } - - juce::String getOutputFilename (const juce::ArgumentList& args) - { - return getOptionValue (args, "--output-filename", {}, "Missing output-filename path argument!").toString(); - } - - std::vector getSampleRates (const juce::ArgumentList& args) - { - juce::StringArray input = juce::StringArray::fromTokens (getOptionValue (args, - "--sample-rates", - juce::String ("44100,48000,96000"), - "Missing sample rate list argument!") - .toString(), - ",", - "\""); - std::vector output; - - for (juce::String sr : input) - output.push_back (sr.getDoubleValue()); - - return output; - } - - std::vector getBlockSizes (const juce::ArgumentList& args) - { - juce::StringArray input = juce::StringArray::fromTokens (getOptionValue (args, - "--block-sizes", - juce::String ("64,128,256,512,1024"), - "Missing block size list argument!") - .toString(), - ",", - "\""); - std::vector output; - - for (juce::String sr : input) - output.push_back (sr.getIntValue()); - - return output; - } - - juce::StringArray getDisabledTests (const juce::ArgumentList& args) - { - const auto value = getOptionValue (args, "--disabled-tests", {}, "Missing disabled-tests path argument!").toString(); - - if (juce::File::isAbsolutePath (value)) - { - const juce::File disabledTestsFile (value); - - juce::StringArray disabledTests; - disabledTestsFile.readLines (disabledTests); - - return disabledTests; - } - - return juce::StringArray::fromTokens (value, ",", ""); - } - - bool isPluginArgument (juce::String arg) - { - juce::AudioPluginFormatManager formatManager; - #if JUCE_VERSION >= 0x08000B - juce::addDefaultFormatsToManager (formatManager); - #else - formatManager.addDefaultFormats(); - #endif - - for (auto format : formatManager.getFormats()) - if (format->fileMightContainThisPluginType (arg)) - return true; - - // The above will check if the file actually exists which isn't really what we want for CLI parsing - if (auto f = juce::File::createFileWithoutCheckingPath (arg); - f.hasFileExtension (".vst3") - #if JUCE_PLUGINHOST_VST - || f.hasFileExtension (".dll") - #endif - ) - return true; - - return false; - } -} - //============================================================================== -struct Option -{ - const char* name; - bool requiresValue; -}; - -static juce::String getEnvironmentVariableName (Option opt) -{ - return juce::String (opt.name).trimCharactersAtStart ("-").replace ("-", "_").toUpperCase(); -} - -static Option possibleOptions[] = -{ - { "--strictness-level", true }, - { "--random-seed", true }, - { "--timeout-ms", true }, - { "--verbose", true }, - { "--skip-gui-tests", false }, - { "--data-file", true }, - { "--output-dir", true }, - { "--output-filename", true }, - { "--repeat", true }, - { "--randomise", false }, - { "--sample-rates", true }, - { "--block-sizes", true }, - { "--rtcheck", false }, -}; - -static juce::StringArray mergeEnvironmentVariables (juce::StringArray args, std::function environmentVariableProvider = [] (const juce::String& name, const juce::String& defaultValue) { return juce::SystemStats::getEnvironmentVariable (name, defaultValue); }) -{ - for (auto arg : possibleOptions) - { - auto envVarName = getEnvironmentVariableName (arg); - auto envVarValue = environmentVariableProvider (envVarName, {}); - - if (envVarValue.isNotEmpty()) - { - const int index = args.indexOf (arg.name); - - if (index != -1) - { - std::cout << "Skipping environment variable " << envVarName << " due to " << arg.name << " set" << std::endl; - continue; - } - - if (arg.requiresValue) - args.insert (0, envVarValue); - - args.insert (0, arg.name); - } - } - - return args; -} - - -//============================================================================== -//============================================================================== -static juce::String getHelpMessage() -{ - const juce::String appName (juce::JUCEApplication::getInstance()->getApplicationName()); - const juce::String juceVersion (juce::SystemStats::getJUCEVersion()); - - return juce::String (R"(//============================================================================== -)" + appName + R"( -)" + juceVersion + R"( - -Description: - Validate plugins to test compatibility with hosts and verify plugin API conformance - -Usage: - --version - Print pluginval version. - --validate [pathToPlugin] - Validates the plugin at the given path. - N.B. the "--validate" flag is optional if the path is the last argument. - This enables you to validate a plugin with simply "pluginval path_to_plugin". - - --sample-rates [list of comma separated sample rates] - If specified, sets the list of sample rates at which tests will be executed - (default=44100,48000,96000) - --block-sizes [list of comma separated block sizes] - If specified, sets the list of block sizes at which tests will be executed - (default=64,128,256,512,1024) - --random-seed [hex or int] - Sets the random seed to use for the tests. Useful for replicating test - environments. - --data-file [pathToFile] - If specified, sets a path to a data file which can be used by tests to - configure themselves. This can be useful for things like known audio output. - - --strictness-level [1-10] - Sets the strictness level to use. A minimum level of 5 (also the default) - is recommended for compatibility. - Higher levels include longer, more thorough tests such as fuzzing. - --strictness-help [level] - Lists all tests that run at the given strictness level (default: 5). - --timeout-ms [numMilliseconds] - Sets a timout which will stop validation with an error if no output from any - test has happened for this number of ms. - By default this is 30s but can be set to "-1" (must be quoted) to never timeout. - --rtcheck [empty, disabled, enabled or relaxed] - Turns on real-time safety checks using rtcheck (macOS and Linux only). - relaxed mode doesn't run the checks for the first processing block as a lot of plugins - use this to allocate or initialise thread-locals (which can allocate) - - --repeat [num repeats] - If specified repeats the tests a given number of times. Note that this does - not delete and re-instantiate the plugin for each repeat. - --randomise - If specified, the tests are run in a random order per repeat. - - --skip-gui-tests - If specified, avoids tests that create GUI windows, which can cause problems - on headless CI systems. - --disabled-tests [pathToFile] - If specified, sets a path to a file that should have the names of disabled - tests on each row. - - --output-dir [pathToDir] - If specified, sets a directory to store the log files. This can be useful - for continuous integration. - --output-filename [filename] - If specified, sets a filename for the log files (within 'output-dir' or - (lacking that) the current directory. - By default, the name is constructed from the plugin metainformation - --verbose - If specified, outputs additional logging information. It can be useful to - turn this off when building with CI to avoid huge log files. - -Exit code: - 0 if all tests complete successfully - 1 if there are any errors - -Additionally, you can specify any of the command line options as environment -variables by removing prefix dashes, converting internal dashes to underscores -and capitalising all letters, a.g. - "--skip-gui-tests" > "SKIP_GUI_TESTS=1" - "--timeout-ms 30000" > "TIMEOUT_MS=30000" -Specifying specific command-line options will override any environment variables -set for that option. -)"); -} - -static juce::String getVersionText() -{ - return juce::String ("pluginval") + " - " + VERSION; -} - static void printStrictnessHelp (int level) { level = juce::jlimit (1, 10, level); @@ -471,209 +176,83 @@ static void runUnitTests() } //============================================================================== -static juce::ArgumentList createCommandLineArgs (juce::String commandLine) -{ - if (commandLine.contains ("strictnessLevel")) - { - std::cout << "!!! WARNING:\n\t\"strictnessLevel\" is deprecated and will be removed in a future version.\n" - << "\tPlease use --strictness-level instead\n\n"; - } - - commandLine = commandLine.replace ("strictnessLevel", "strictness-level") - .replace ("-NSDocumentRevisionsDebugMode YES", "") - .trim(); - - const auto exe = juce::File::getSpecialLocation (juce::File::currentExecutableFile); - - juce::StringArray args; - args.addTokens (commandLine, true); - args = mergeEnvironmentVariables (args); - args.trim(); - - for (auto& s : args) - s = s.unquoted(); - - // If only a plugin path is supplied as the last arg, add an implicit --validate - // option for it so the rest of the CLI works - juce::ArgumentList argList (exe.getFullPathName(), args); - - if (argList.size() > 0) - { - const bool hasValidateOrOtherCommand = argList.containsOption ("--validate") - || argList.containsOption ("--help|-h") - || argList.containsOption ("--version") - || argList.containsOption ("--run-tests"); - - if (! hasValidateOrOtherCommand) - if (isPluginArgument (argList.arguments.getLast().text)) - argList.arguments.insert (argList.arguments.size() - 1, { "--validate" }); - } - - return argList; -} - -static void performCommandLine (CommandLineValidator& validator, const juce::ArgumentList& args) -{ - hideDockIcon(); - - juce::ConsoleApplication cli; - cli.addVersionCommand ("--version", getVersionText()); - cli.addHelpCommand ("--help|-h", getHelpMessage(), true); - cli.addCommand ({ "--validate", - "--validate [pathToPlugin]", - "Validates the file (or IDs for AUs).", juce::String(), - [&validator] (const auto& validatorArgs) - { - auto [fileOrIDToValidate, options] = parseCommandLine (validatorArgs); - validator.validate (fileOrIDToValidate, options); - }}); - cli.addCommand ({ "--run-tests", - "--run-tests", - "Runs the internal unit tests.", juce::String(), - [] (const auto&) { runUnitTests(); }}); - cli.addCommand ({ "--strictness-help", - "--strictness-help [level]", - "Lists all tests that run at the given strictness level.", juce::String(), - [] (const auto& args) - { - int level = 5; - auto arg = getArgumentAfterOption (args, "--strictness-help"); - if (arg.text.isNotEmpty() && ! arg.isShortOption() && ! arg.isLongOption()) - level = arg.text.getIntValue(); - printStrictnessHelp (level); - }}); - - if (const auto retValue = cli.findAndRunCommand (args); retValue != 0) - { - juce::JUCEApplication::getInstance()->setApplicationReturnValue (retValue); - juce::JUCEApplication::getInstance()->quit(); - } - - // --validate runs async so will quit itself when done - if (! args.containsOption ("--validate")) - juce::JUCEApplication::getInstance()->quit(); -} - //============================================================================== -void performCommandLine (CommandLineValidator& validator, const juce::String& commandLine) +std::pair parseCommandLine (const juce::String& commandLine) { - performCommandLine (validator, createCommandLineArgs (commandLine)); + const auto settings = settings_parser::parse (commandLine); + return { juce::String (settings.validatePath), settings.toPluginTestOptions() }; } -bool shouldPerformCommandLine (const juce::String& commandLine) +juce::StringArray createCommandLine (juce::String fileOrID, PluginTests::Options options) { - const auto args = createCommandLineArgs (commandLine); - return args.containsOption ("--help|-h") - || args.containsOption ("--version") - || args.containsOption ("--validate") - || args.containsOption ("--run-tests") - || args.containsOption ("--strictness-help"); + return settings_parser::createChildProcessCommandLine (fileOrID, options); } //============================================================================== -//============================================================================== -std::pair parseCommandLine (const juce::ArgumentList& args) -{ - auto fileOrID = getOptionValue (args, "--validate", "", "Expected a plugin path for the --validate option").toString(); - - // in the case of a path (vs. ID), grab the full path - // getCurrentWorkingDirectory is needed to handle relative paths - // It preserves absolute paths and first checks for ~ on Mac/Windows - if (fileOrID.contains ("~") || fileOrID.contains (".")) - fileOrID = juce::File::getCurrentWorkingDirectory().getChildFile(fileOrID).getFullPathName(); - - PluginTests::Options options; - options.strictnessLevel = getStrictnessLevel (args); - options.randomSeed = getRandomSeed (args); - options.timeoutMs = getTimeout (args); - options.verbose = args.containsOption ("--verbose"); - options.numRepeats = getNumRepeats (args); - options.randomiseTestOrder = args.containsOption ("--randomise"); - options.dataFile = getDataFile (args); - options.outputDir = getOutputDir (args); - options.outputFilename = getOutputFilename (args); - options.withGUI = ! args.containsOption ("--skip-gui-tests"); - options.disabledTests = getDisabledTests (args); - options.sampleRates = getSampleRates (args); - options.blockSizes = getBlockSizes (args); - options.realtimeCheck = magic_enum::enum_cast (getOptionValue (args, "--rtcheck", "", "Expected one of [disabled, enabled, relaxed]").toString().toStdString()) - .value_or (RealtimeCheck::disabled); - - return { fileOrID, options }; -} - -std::pair parseCommandLine (const juce::String& cmd) -{ - return parseCommandLine (createCommandLineArgs (cmd)); -} - -juce::StringArray createCommandLine (juce::String fileOrID, PluginTests::Options options) +void performCommandLine (CommandLineValidator& validator, const juce::String& commandLine) { - juce::StringArray args (juce::File::getSpecialLocation (juce::File::currentExecutableFile).getFullPathName()); - const PluginTests::Options defaults; - - if (options.strictnessLevel != defaults.strictnessLevel) - args.addArray ({ "--strictness-level", juce::String (options.strictnessLevel) }); - - if (options.randomSeed != defaults.randomSeed) - args.addArray ({ "--random-seed", juce::String (options.randomSeed) }); - - if (options.timeoutMs != defaults.timeoutMs) - args.addArray ({ "--timeout-ms", juce::String (options.timeoutMs) }); - - if (options.verbose) - args.add ("--verbose"); - - if (! options.withGUI) - args.add ("--skip-gui-tests"); - - if (options.numRepeats != defaults.numRepeats) - args.addArray ({ "--repeat", juce::String (options.numRepeats) }); - - if (options.randomiseTestOrder) - args.add ("--randomise"); + hideDockIcon(); - if (options.dataFile != defaults.dataFile) - args.addArray ({ "--data-file", options.dataFile.getFullPathName() }); + auto& app = *juce::JUCEApplication::getInstance(); + const auto tokens = settings_parser::preprocess (commandLine); - if (options.outputDir != defaults.outputDir) - args.addArray ({ "--output-dir", options.outputDir.getFullPathName() }); + if (tokens.contains ("--help") || tokens.contains ("-h")) + { + settings_parser::printHelp (app.getApplicationName()); + app.quit(); + return; + } - if (options.outputFilename != defaults.outputFilename) - args.addArray ({ "--output-filename", options.outputFilename }); + if (tokens.contains ("--version")) + { + std::cout << settings_parser::getVersionString() << std::endl; + app.quit(); + return; + } - if (options.disabledTests != defaults.disabledTests) - args.addArray ({ "--disabled-tests", options.disabledTests.joinIntoString (",") }); + if (tokens.contains ("--run-tests")) + { + runUnitTests(); + app.quit(); + return; + } - if (! options.sampleRates.empty()) + if (tokens.contains ("--strictness-help")) { - juce::StringArray sampleRates; + int level = 5; - for (auto rate : options.sampleRates) - sampleRates.add (juce::String (rate)); + if (const auto idx = tokens.indexOf ("--strictness-help"); idx >= 0 && idx + 1 < tokens.size()) + if (const auto next = tokens[idx + 1]; ! next.startsWith ("-")) + level = next.getIntValue(); - args.addArray ({ "--sample-rates", sampleRates.joinIntoString (",") }); + printStrictnessHelp (level); + app.quit(); + return; } - if (! options.blockSizes.empty()) + // Otherwise this is a validation run (explicit or implicit --validate) + try { - juce::StringArray blockSizes; + auto [fileOrID, options] = parseCommandLine (commandLine); - for (auto size : options.blockSizes) - blockSizes.add (juce::String (size)); + if (fileOrID.isEmpty()) + { + exitWithError ("*** FAILED: No plugin path or ID specified to validate"); + return; + } - args.addArray ({ "--block-sizes", blockSizes.joinIntoString (",") }); + // --validate runs async so will quit itself when done + validator.validate (fileOrID, options); } - - if (auto rtCheckMode = options.realtimeCheck; - rtCheckMode != RealtimeCheck::disabled) + catch (const std::exception& e) { - args.addArray ({ "--rtcheck", std::string (magic_enum::enum_name (rtCheckMode)) }); + exitWithError (juce::String ("*** FAILED: ") + e.what()); } +} - args.addArray ({ "--validate", fileOrID }); - - return args; +bool shouldPerformCommandLine (const juce::String& commandLine) +{ + return settings_parser::isCommandLine (settings_parser::preprocess (commandLine)); } //============================================================================== diff --git a/Source/CommandLine.h b/Source/CommandLine.h index 644eb55..c19f69f 100644 --- a/Source/CommandLine.h +++ b/Source/CommandLine.h @@ -34,6 +34,8 @@ void performCommandLine (CommandLineValidator&, const juce::String& commandLine) bool shouldPerformCommandLine (const juce::String& commandLine); //============================================================================== +/** Parses a command line into the plugin path/ID and resolved test options. */ std::pair parseCommandLine (const juce::String&); -std::pair parseCommandLine (const juce::ArgumentList&); + +/** Serialises options for the child validation process. */ juce::StringArray createCommandLine (juce::String fileOrID, PluginTests::Options); diff --git a/Source/CommandLineTests.cpp b/Source/CommandLineTests.cpp index 9be4521..9d2ba44 100644 --- a/Source/CommandLineTests.cpp +++ b/Source/CommandLineTests.cpp @@ -20,146 +20,265 @@ struct CommandLineTests : public juce::UnitTest { } - void runTest() override + // A deterministic, empty environment so tests don't pick up the host's env vars. + static settings_parser::EnvProvider emptyEnv() { + return [] (const juce::String&) { return juce::String(); }; + } - beginTest ("Merge environment variables"); - { - juce::StringPairArray envVars; - envVars.set ("STRICTNESS_LEVEL", "5"); - envVars.set ("RANDOM_SEED", "1234"); - envVars.set ("TIMEOUT_MS", "30000"); - envVars.set ("VERBOSE", "1"); - envVars.set ("REPEAT", "10"); - envVars.set ("RANDOMISE", "1"); - envVars.set ("SKIP_GUI_TESTS", "1"); - envVars.set ("DATA_FILE", ""); - envVars.set ("OUTPUT_DIR", ""); - - const auto merged = mergeEnvironmentVariables (juce::String(), [&envVars] (const juce::String& n, const juce::String& def) { return envVars.getValue (n, def); }).joinIntoString (" "); - expect (merged.contains ("--strictness-level 5")); - expect (merged.contains ("--random-seed 1234")); - expect (merged.contains ("--timeout-ms 30000")); - expect (merged.contains ("--verbose")); - expect (merged.contains ("--repeat 10")); - expect (merged.contains ("--randomise")); - expect (merged.contains ("--skip-gui-tests")); - expect (merged.contains ("--data-file ")); - expect (merged.contains ("--output-dir ")); - } + static PluginvalSettings parse (const juce::String& cmd, settings_parser::EnvProvider env) + { + return settings_parser::resolveSettings (settings_parser::preprocess (cmd), env); + } + void runTest() override + { beginTest ("Command line defaults"); { - juce::ArgumentList args ({}, ""); - expectEquals (getStrictnessLevel (args), 5); - expectEquals (getRandomSeed (args), (juce::int64) 0); - expectEquals (getTimeout (args), (juce::int64) 30000); - expectEquals (getNumRepeats (args), 1); - expectEquals (getOptionValue (args, "--data-file", {}, "Missing data-file path argument!").toString(), juce::String()); - expectEquals (getOptionValue (args, "--output-dir", {}, "Missing output-dir path argument!").toString(), juce::String()); + const auto opts = parse ("", emptyEnv()).toPluginTestOptions(); + expectEquals (opts.strictnessLevel, 5); + expectEquals (opts.randomSeed, (juce::int64) 0); + expectEquals (opts.timeoutMs, (juce::int64) 30000); + expectEquals (opts.numRepeats, 1); + expect (opts.verbose == false); + expect (opts.randomiseTestOrder == false); + expect (opts.withGUI == true); + expect (opts.realtimeCheck == RealtimeCheck::disabled); + expect (opts.dataFile == juce::File()); + expect (opts.outputDir == juce::File()); + expect (opts.sampleRates == PluginvalSettings::defaultSampleRates()); + expect (opts.blockSizes == PluginvalSettings::defaultBlockSizes()); } beginTest ("Command line parser"); { - juce::ArgumentList args ({}, "--strictness-level 7 --random-seed 1234 --timeout-ms 20000 --repeat 11 --data-file /path/to/file --output-dir /path/to/dir --validate /path/to/plugin"); - expectEquals (getStrictnessLevel (args), 7); - expectEquals (getRandomSeed (args), (juce::int64) 1234); - expectEquals (getTimeout (args), (juce::int64) 20000); - expectEquals (getNumRepeats (args), 11); - expectEquals (getOptionValue (args, "--data-file", {}, "Missing data-file path argument!").toString(),juce::String ("/path/to/file")); - expectEquals (getOptionValue (args, "--output-dir", {}, "Missing output-dir path argument!").toString(),juce::String ("/path/to/dir")); - expectEquals (getOptionValue (args, "--validate", {}, "Missing validate argument!").toString(),juce::String ("/path/to/plugin")); + const auto settings = parse ("--strictness-level 7 --random-seed 1234 --timeout-ms 20000 --repeat 11 " + "--data-file /path/to/file --output-dir /path/to/dir --validate /path/to/plugin", + emptyEnv()); + const auto opts = settings.toPluginTestOptions(); + expectEquals (opts.strictnessLevel, 7); + expectEquals (opts.randomSeed, (juce::int64) 1234); + expectEquals (opts.timeoutMs, (juce::int64) 20000); + expectEquals (opts.numRepeats, 11); + expectEquals (opts.dataFile.getFullPathName(), juce::String ("/path/to/file")); + expectEquals (opts.outputDir.getFullPathName(), juce::String ("/path/to/dir")); + expectEquals (juce::String (settings.validatePath), juce::String ("/path/to/plugin")); + } + + beginTest ("Command line random (hex and int)"); + { + expectEquals (parse ("--random-seed 0x7f2da1 --validate x", emptyEnv()).randomSeed, (juce::int64) 8334753); + expectEquals (parse ("--random-seed 0x692bc1f --validate x", emptyEnv()).randomSeed, (juce::int64) 110279711); + expectEquals (parse ("--random-seed 1234 --validate x", emptyEnv()).randomSeed, (juce::int64) 1234); + } + + beginTest ("Comma-separated lists"); + { + const auto opts = parse ("--sample-rates 22050,44100 --block-sizes 32,64,128 --validate x", emptyEnv()).toPluginTestOptions(); + expect (opts.sampleRates == std::vector ({ 22050.0, 44100.0 })); + expect (opts.blockSizes == std::vector ({ 32, 64, 128 })); + } + + beginTest ("rtcheck enum parsing"); + { + expect (parse ("--rtcheck relaxed --validate x", emptyEnv()).realtimeCheck == RealtimeCheck::relaxed); + expect (parse ("--rtcheck enabled --validate x", emptyEnv()).realtimeCheck == RealtimeCheck::enabled); + expect (parse ("--validate x", emptyEnv()).realtimeCheck == RealtimeCheck::disabled); } beginTest ("Handles an absolute path to the plugin"); { const auto homeDir = juce::File::getSpecialLocation (juce::File::userHomeDirectory).getFullPathName(); - const auto commandLineString = "--validate " + homeDir + "/path/to/MyPlugin"; - const auto args = createCommandLineArgs (commandLineString); - expectEquals (parseCommandLine (args).first, homeDir + "/path/to/MyPlugin"); + expectEquals (juce::String (parse ("--validate " + homeDir + "/path/to/MyPlugin", emptyEnv()).validatePath), + homeDir + "/path/to/MyPlugin"); } beginTest ("Handles a quoted absolute path to the plugin"); { const auto homeDir = juce::File::getSpecialLocation (juce::File::userHomeDirectory).getFullPathName(); const auto pathToQuote = homeDir + "/path/to/MyPlugin"; - const auto commandLineString = "--validate " + pathToQuote.quoted(); - const auto args = createCommandLineArgs (commandLineString); - expectEquals (parseCommandLine (args).first, homeDir + "/path/to/MyPlugin"); + expectEquals (juce::String (parse ("--validate " + pathToQuote.quoted(), emptyEnv()).validatePath), + homeDir + "/path/to/MyPlugin"); } beginTest ("Handles a relative path"); { const auto currentDir = juce::File::getCurrentWorkingDirectory(); - const auto args = createCommandLineArgs ("--validate MyPlugin.vst3"); - expectEquals (parseCommandLine (args).first, currentDir.getChildFile ("MyPlugin.vst3").getFullPathName()); + expectEquals (juce::String (parse ("--validate MyPlugin.vst3", emptyEnv()).validatePath), + currentDir.getChildFile ("MyPlugin.vst3").getFullPathName()); } beginTest ("Handles a quoted relative path with spaces to the plugin"); { const auto currentDir = juce::File::getCurrentWorkingDirectory(); - const auto args = createCommandLineArgs (R"(--validate "My Plugin.vst3")"); - expectEquals (parseCommandLine (args).first, currentDir.getChildFile ("My Plugin.vst3").getFullPathName()); + expectEquals (juce::String (parse (R"(--validate "My Plugin.vst3")", emptyEnv()).validatePath), + currentDir.getChildFile ("My Plugin.vst3").getFullPathName()); } - #if !JUCE_WINDOWS - + #if !JUCE_WINDOWS beginTest ("Handles a relative path with ./ to the plugin"); { const auto currentDir = juce::File::getCurrentWorkingDirectory().getFullPathName(); - const auto commandLineString = "--validate ./path/to/MyPlugin"; - const auto args = createCommandLineArgs(commandLineString); - expectEquals (parseCommandLine (args).first, currentDir + "/path/to/MyPlugin"); + expectEquals (juce::String (parse ("--validate ./path/to/MyPlugin", emptyEnv()).validatePath), + currentDir + "/path/to/MyPlugin"); } beginTest ("Handles a home directory relative path to the plugin"); { - const auto commandLineString = "--validate ~/path/to/MyPlugin"; - const auto args = createCommandLineArgs(commandLineString); - expectEquals (parseCommandLine (args).first, juce::File::getSpecialLocation (juce::File::userHomeDirectory).getFullPathName() + "/path/to/MyPlugin"); + expectEquals (juce::String (parse ("--validate ~/path/to/MyPlugin", emptyEnv()).validatePath), + juce::File::getSpecialLocation (juce::File::userHomeDirectory).getFullPathName() + "/path/to/MyPlugin"); } beginTest ("Handles quoted strings, spaces, and home directory relative path to the plugin"); { - const auto commandLineString = R"(--data-file "~/path/to/My File" --output-dir "~/path/to/My Directory" --validate "~/path/to/My Plugin")"; - const auto args = createCommandLineArgs(commandLineString); - expectEquals (parseCommandLine (args).first, juce::File::getSpecialLocation (juce::File::userHomeDirectory).getFullPathName() + "/path/to/My Plugin"); + const auto cmd = R"(--data-file "~/path/to/My File" --output-dir "~/path/to/My Directory" --validate "~/path/to/My Plugin")"; + expectEquals (juce::String (parse (cmd, emptyEnv()).validatePath), + juce::File::getSpecialLocation (juce::File::userHomeDirectory).getFullPathName() + "/path/to/My Plugin"); } - #endif + #endif beginTest ("Implicit validate with a relative path"); { const auto currentDir = juce::File::getCurrentWorkingDirectory(); - const auto args = createCommandLineArgs ("MyPlugin.vst3"); - expectEquals (parseCommandLine (args).first, currentDir.getChildFile ("MyPlugin.vst3").getFullPathName()); + expectEquals (juce::String (parse ("MyPlugin.vst3", emptyEnv()).validatePath), + currentDir.getChildFile ("MyPlugin.vst3").getFullPathName()); } beginTest ("Doesn't alter component IDs"); { - const auto commandLineString = "--validate MyPluginID"; - const auto args = createCommandLineArgs(commandLineString); - expectEquals (parseCommandLine (args).first,juce::String ("MyPluginID")); + expectEquals (juce::String (parse ("--validate MyPluginID", emptyEnv()).validatePath), juce::String ("MyPluginID")); } - beginTest ("Command line random"); + beginTest ("Allows for other options after explicit --validate"); { - expectEquals (getRandomSeed (juce::ArgumentList ({}, "--random-seed 0x7f2da1")), (juce::int64) 8334753); - expectEquals (getRandomSeed (juce::ArgumentList ({}, "--random-seed 0x692bc1f")), (juce::int64) 110279711); + const auto currentDir = juce::File::getCurrentWorkingDirectory(); + const auto settings = parse ("--validate MyPlugin.vst3 --randomise", emptyEnv()); + expectEquals (juce::String (settings.validatePath), currentDir.getChildFile ("MyPlugin.vst3").getFullPathName()); + expect (settings.randomiseTestOrder); } - beginTest ("Implicit validate options"); + beginTest ("Should perform command line"); { juce::TemporaryFile temp ("path_to_file.vst3"); expect (temp.getFile().create()); expect (shouldPerformCommandLine (temp.getFile().getFullPathName())); + expect (shouldPerformCommandLine ("--run-tests")); + expect (shouldPerformCommandLine ("--version")); + expect (! shouldPerformCommandLine ("")); } - beginTest ("Allows for other options after explicit --validate"); + beginTest ("Environment variables"); { - const auto currentDir = juce::File::getCurrentWorkingDirectory(); - const auto args = createCommandLineArgs ("--validate MyPlugin.vst3 --randomise"); - expectEquals (parseCommandLine (args).first, currentDir.getChildFile ("MyPlugin.vst3").getFullPathName()); - expect (parseCommandLine(args).second.randomiseTestOrder); + std::map envVars { + { "STRICTNESS_LEVEL", "7" }, + { "RANDOM_SEED", "1234" }, + { "TIMEOUT_MS", "20000" }, + { "VERBOSE", "1" }, + { "REPEAT", "11" }, + { "RANDOMISE", "1" }, + { "SKIP_GUI_TESTS", "1" }, + { "DATA_FILE", "/path/to/file" }, + { "OUTPUT_DIR", "/path/to/dir" }, + { "SAMPLE_RATES", "22050,44100" }, + { "BLOCK_SIZES", "32,64" }, + { "RTCHECK", "relaxed" }, + }; + + settings_parser::EnvProvider env = [envVars] (const juce::String& n) + { + const auto it = envVars.find (n); + return it != envVars.end() ? it->second : juce::String(); + }; + + const auto opts = parse ("--validate x", env).toPluginTestOptions(); + expectEquals (opts.strictnessLevel, 7); + expectEquals (opts.randomSeed, (juce::int64) 1234); + expectEquals (opts.timeoutMs, (juce::int64) 20000); + expect (opts.verbose); + expectEquals (opts.numRepeats, 11); + expect (opts.randomiseTestOrder); + expect (opts.withGUI == false); + expect (opts.sampleRates == std::vector ({ 22050.0, 44100.0 })); + expect (opts.blockSizes == std::vector ({ 32, 64 })); + expect (opts.realtimeCheck == RealtimeCheck::relaxed); + } + + beginTest ("Command line overrides environment variables"); + { + settings_parser::EnvProvider env = [] (const juce::String& n) + { + return n == "STRICTNESS_LEVEL" ? juce::String ("3") : juce::String(); + }; + + expectEquals (parse ("--validate x", env).strictnessLevel, 3); // env only + expectEquals (parse ("--strictness-level 9 --validate x", env).strictnessLevel, 9); // CLI wins + } + + beginTest ("Config file and precedence (CLI > env > config > defaults)"); + { + juce::TemporaryFile configFile (".json"); + configFile.getFile().replaceWithText (R"({ "strictnessLevel": 2, "timeoutMs": 12345, "numRepeats": 4 })"); + const auto cfg = "--config " + configFile.getFile().getFullPathName().quoted(); + + // config alone + { + const auto s = parse (cfg + " --validate x", emptyEnv()); + expectEquals (s.strictnessLevel, 2); + expectEquals (s.timeoutMs, (juce::int64) 12345); + expectEquals (s.numRepeats, 4); + } + + // env overrides config + { + settings_parser::EnvProvider env = [] (const juce::String& n) + { return n == "STRICTNESS_LEVEL" ? juce::String ("6") : juce::String(); }; + + const auto s = parse (cfg + " --validate x", env); + expectEquals (s.strictnessLevel, 6); // env beats config + expectEquals (s.timeoutMs, (juce::int64) 12345); // still from config + } + + // CLI overrides env and config + { + settings_parser::EnvProvider env = [] (const juce::String& n) + { return n == "STRICTNESS_LEVEL" ? juce::String ("6") : juce::String(); }; + + const auto s = parse (cfg + " --strictness-level 9 --validate x", env); + expectEquals (s.strictnessLevel, 9); + expectEquals (s.timeoutMs, (juce::int64) 12345); + } + } + + beginTest ("Child-process command line round-trip"); + { + PluginTests::Options opts; + opts.strictnessLevel = 8; + opts.randomSeed = 8334753; + opts.timeoutMs = 15000; + opts.verbose = true; + opts.numRepeats = 3; + opts.randomiseTestOrder = true; + opts.withGUI = false; + opts.outputFilename = "log.txt"; + opts.disabledTests = juce::StringArray ({ "Test A", "Test B" }); + opts.sampleRates = { 44100.0, 96000.0 }; + opts.blockSizes = { 64, 512 }; + opts.realtimeCheck = RealtimeCheck::relaxed; + + const juce::String fileOrID = "/some/dir/MyPlugin.vst3"; + + const auto args = createCommandLine (fileOrID, opts); + + juce::StringArray childArgs (args); + childArgs.remove (0); // drop the executable path + const auto childCommandLine = childArgs.joinIntoString (" "); + + const auto [fileOrID2, opts2] = parseCommandLine (childCommandLine); + + auto expected = PluginvalSettings::fromPluginTestOptions (opts, fileOrID); + expected.validatePath = settings_parser::resolvePluginPath (fileOrID).toStdString(); + + expect (PluginvalSettings::fromPluginTestOptions (opts2, fileOrID2) == expected); } } }; diff --git a/Source/PluginvalSettings.h b/Source/PluginvalSettings.h new file mode 100644 index 0000000..e8a8255 --- /dev/null +++ b/Source/PluginvalSettings.h @@ -0,0 +1,142 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#pragma once + +#include "PluginTests.h" +#include + +#include +#include +#include + +//============================================================================== +/** Maps the RealtimeCheck enum to/from its JSON string representation. + An unrecognised/empty string deserialises to the first entry (disabled), + preserving the historical "empty -> disabled" behaviour. +*/ +NLOHMANN_JSON_SERIALIZE_ENUM (RealtimeCheck, +{ + { RealtimeCheck::disabled, "disabled" }, + { RealtimeCheck::enabled, "enabled" }, + { RealtimeCheck::relaxed, "relaxed" }, +}) + +//============================================================================== +/** + The single, parser-agnostic settings struct. + + Every field uses std types so it can be (de)serialised by nlohmann/json and + is decoupled from JUCE. JSON keys mirror the field names below. Conversion to + the JUCE-flavoured PluginTests::Options happens only at the boundary via + toPluginTestOptions(). +*/ +struct PluginvalSettings +{ + int strictnessLevel = 5; + std::int64_t randomSeed = 0; + std::int64_t timeoutMs = 30000; + bool verbose = false; + int numRepeats = 1; + bool randomiseTestOrder = false; + bool skipGuiTests = false; + std::string dataFile; + std::string outputDir; + std::string outputFilename; + std::vector disabledTests; + std::vector sampleRates; + std::vector blockSizes; + RealtimeCheck realtimeCheck = RealtimeCheck::disabled; + std::string validatePath; + + bool operator== (const PluginvalSettings&) const = default; + + //============================================================================== + /** The default sample rates used when none are specified. */ + static std::vector defaultSampleRates() { return { 44100.0, 48000.0, 96000.0 }; } + /** The default block sizes used when none are specified. */ + static std::vector defaultBlockSizes() { return { 64, 128, 256, 512, 1024 }; } + + //============================================================================== + /** Converts to the JUCE-flavoured options consumed by PluginTests. */ + PluginTests::Options toPluginTestOptions() const + { + PluginTests::Options o; + o.strictnessLevel = juce::jlimit (1, 10, strictnessLevel); + o.randomSeed = randomSeed; + o.timeoutMs = timeoutMs; + o.verbose = verbose; + o.numRepeats = juce::jmax (1, numRepeats); + o.randomiseTestOrder = randomiseTestOrder; + o.withGUI = ! skipGuiTests; + o.dataFile = juce::String (dataFile); + o.outputDir = juce::String (outputDir); + o.outputFilename = juce::String (outputFilename); + o.disabledTests = toStringArray (disabledTests); + o.sampleRates = sampleRates.empty() ? defaultSampleRates() : sampleRates; + o.blockSizes = blockSizes.empty() ? defaultBlockSizes() : blockSizes; + o.realtimeCheck = realtimeCheck; + return o; + } + + /** Builds settings from a resolved PluginTests::Options + plugin path. + Used to serialise options for the child validation process. + */ + static PluginvalSettings fromPluginTestOptions (const PluginTests::Options& o, const juce::String& fileOrID) + { + PluginvalSettings s; + s.strictnessLevel = o.strictnessLevel; + s.randomSeed = o.randomSeed; + s.timeoutMs = o.timeoutMs; + s.verbose = o.verbose; + s.numRepeats = o.numRepeats; + s.randomiseTestOrder = o.randomiseTestOrder; + s.skipGuiTests = ! o.withGUI; + s.dataFile = o.dataFile.getFullPathName().toStdString(); + s.outputDir = o.outputDir.getFullPathName().toStdString(); + s.outputFilename = o.outputFilename.toStdString(); + s.disabledTests = fromStringArray (o.disabledTests); + s.sampleRates = o.sampleRates; + s.blockSizes = o.blockSizes; + s.realtimeCheck = o.realtimeCheck; + s.validatePath = fileOrID.toStdString(); + return s; + } + +private: + static juce::StringArray toStringArray (const std::vector& v) + { + juce::StringArray a; + for (const auto& s : v) + a.add (juce::String (s)); + return a; + } + + static std::vector fromStringArray (const juce::StringArray& a) + { + std::vector v; + for (const auto& s : a) + v.push_back (s.toStdString()); + return v; + } +}; + +//============================================================================== +// to_json/from_json with per-field defaults: keys missing from the JSON fall +// back to a default-constructed PluginvalSettings rather than value-zero. This +// makes "defaults" a free layer and lets sparse JSON layers merge cleanly. +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT (PluginvalSettings, + strictnessLevel, randomSeed, timeoutMs, verbose, numRepeats, + randomiseTestOrder, skipGuiTests, dataFile, outputDir, outputFilename, + disabledTests, sampleRates, blockSizes, realtimeCheck, validatePath) diff --git a/Source/SettingsParser.cpp b/Source/SettingsParser.cpp new file mode 100644 index 0000000..e08041f --- /dev/null +++ b/Source/SettingsParser.cpp @@ -0,0 +1,363 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#include "SettingsParser.h" +#include "SettingsSerializer.h" + +#include + +#include +#include +#include +#include +#include + +namespace settings_parser +{ + //============================================================================== + // The magic_args view of the flat command-line options. Everything is parsed + // as a raw optional/flag; typed coercion happens when building the JSON layer. + struct PluginvalCliArgs + { + magic_args::option> validate + { .mName = "validate", .mHelp = "Validates the plugin at the given path (or AU id). Optional if the path is the last argument." }; + magic_args::option> strictnessLevel + { .mName = "strictness-level", .mHelp = "Strictness level 1-10 (default 5, minimum 5 recommended)." }; + magic_args::option> randomSeed + { .mName = "random-seed", .mHelp = "Random seed (hex 0x.. or int) for replicable test runs." }; + magic_args::option> timeoutMs + { .mName = "timeout-ms", .mHelp = "Test timeout in ms (default 30000, -1 to never timeout)." }; + magic_args::option> repeat + { .mName = "repeat", .mHelp = "Number of times to repeat the tests." }; + magic_args::flag randomise + { .mName = "randomise", .mHelp = "Run the tests in a random order per repeat." }; + magic_args::flag verbose + { .mName = "verbose", .mHelp = "Output additional logging information." }; + magic_args::flag skipGuiTests + { .mName = "skip-gui-tests", .mHelp = "Avoid tests that create GUI windows (for headless CI)." }; + magic_args::option> sampleRates + { .mName = "sample-rates", .mHelp = "Comma-separated sample rates (default 44100,48000,96000)." }; + magic_args::option> blockSizes + { .mName = "block-sizes", .mHelp = "Comma-separated block sizes (default 64,128,256,512,1024)." }; + magic_args::option> dataFile + { .mName = "data-file", .mHelp = "Path to a data file tests can use to configure themselves." }; + magic_args::option> outputDir + { .mName = "output-dir", .mHelp = "Directory in which to write the log files." }; + magic_args::option> outputFilename + { .mName = "output-filename", .mHelp = "Filename to write logs into." }; + magic_args::option> disabledTests + { .mName = "disabled-tests", .mHelp = "Comma-separated test names, or a path to a file listing them." }; + magic_args::option> rtcheck + { .mName = "rtcheck", .mHelp = "Real-time safety checks: disabled, enabled or relaxed." }; + magic_args::option> config + { .mName = "config", .mHelp = "Path to a JSON file of settings (overridden by env vars and CLI options)." }; + }; + + //============================================================================== + static magic_args::program_info makeProgramInfo() + { + return { + .mDescription = "Validate plugins to test compatibility with hosts and verify plugin API conformance", + .mVersion = getVersionString().toStdString(), + .mExamples = { + "pluginval --strictness-level 5 --validate path/to/plugin", + "pluginval path/to/plugin", + "pluginval --config settings.json --validate path/to/plugin", + }, + }; + } + + static juce::String getTrailerText() + { + return juce::String ( +R"(Other commands: + --version Print the pluginval version. + --run-tests Run the internal unit tests. + --strictness-help [level] List all tests that run at the given strictness level. + +Exit code: + 0 if all tests complete successfully + 1 if there are any errors + +Additionally, you can specify any of the command line options as environment +variables by removing prefix dashes, converting internal dashes to underscores +and capitalising all letters, e.g. + "--skip-gui-tests" > "SKIP_GUI_TESTS=1" + "--timeout-ms 30000" > "TIMEOUT_MS=30000" +Precedence is command-line options > environment variables > --config file. +)"); + } + + //============================================================================== + juce::String systemEnv (const juce::String& name) + { + return juce::SystemStats::getEnvironmentVariable (name, {}); + } + + juce::String getVersionString() + { + return juce::String ("pluginval") + " - " + VERSION; + } + + //============================================================================== + namespace + { + bool isPluginArgument (const juce::String& arg) + { + juce::AudioPluginFormatManager formatManager; + #if JUCE_VERSION >= 0x08000B + juce::addDefaultFormatsToManager (formatManager); + #else + formatManager.addDefaultFormats(); + #endif + + for (auto format : formatManager.getFormats()) + if (format->fileMightContainThisPluginType (arg)) + return true; + + // The above checks the file exists, which isn't what we want for CLI parsing + if (auto f = juce::File::createFileWithoutCheckingPath (arg); + f.hasFileExtension (".vst3") + #if JUCE_PLUGINHOST_VST + || f.hasFileExtension (".dll") + #endif + ) + return true; + + return false; + } + + juce::String valueForOption (const juce::StringArray& tokens, juce::StringRef option) + { + const juce::String prefix = juce::String (option) + "="; + + for (int i = 0; i < tokens.size(); ++i) + { + if (tokens[i] == option) + return i + 1 < tokens.size() ? tokens[i + 1] : juce::String(); + + if (tokens[i].startsWith (prefix)) + return tokens[i].substring (prefix.length()); + } + + return {}; + } + + bool hasOption (const juce::StringArray& tokens, juce::StringRef option) + { + const juce::String prefix = juce::String (option) + "="; + + for (const auto& t : tokens) + if (t == option || t.startsWith (prefix)) + return true; + + return false; + } + + juce::String decodeBase64 (const juce::String& b64) + { + juce::MemoryOutputStream mos; + juce::Base64::convertFromBase64 (mos, b64); + return mos.toString(); + } + + // Builds the sparse CLI JSON layer by parsing tokens with magic_args. + nlohmann::json parseCliLayer (const juce::StringArray& tokens) + { + std::vector storage; + storage.reserve ((size_t) tokens.size() + 1); + storage.emplace_back ("pluginval"); + + for (const auto& t : tokens) + storage.push_back (t.toStdString()); + + std::vector views (storage.begin(), storage.end()); + + const auto parsed = magic_args::parse (std::span (views), + makeProgramInfo()); + + auto j = nlohmann::json::object(); + + if (! parsed) // HelpRequested / VersionRequested / parse error: contribute nothing + return j; + + const auto& a = *parsed; + + if (a.validate.has_value()) j["validatePath"] = *a.validate; + if (a.strictnessLevel.has_value()) j["strictnessLevel"] = *a.strictnessLevel; + if (a.randomSeed.has_value()) j["randomSeed"] = settings_serializer::parseRandomSeed (juce::String (*a.randomSeed)); + if (a.timeoutMs.has_value()) j["timeoutMs"] = *a.timeoutMs; + if (a.repeat.has_value()) j["numRepeats"] = *a.repeat; + if (a.verbose) j["verbose"] = true; + if (a.randomise) j["randomiseTestOrder"] = true; + if (a.skipGuiTests) j["skipGuiTests"] = true; + if (a.sampleRates.has_value()) j["sampleRates"] = settings_serializer::commaToDoubleArray (juce::String (*a.sampleRates)); + if (a.blockSizes.has_value()) j["blockSizes"] = settings_serializer::commaToIntArray (juce::String (*a.blockSizes)); + if (a.dataFile.has_value()) j["dataFile"] = *a.dataFile; + if (a.outputDir.has_value()) j["outputDir"] = *a.outputDir; + if (a.outputFilename.has_value()) j["outputFilename"] = *a.outputFilename; + if (a.disabledTests.has_value()) j["disabledTests"] = settings_serializer::disabledTestsToArray (juce::String (*a.disabledTests)); + if (a.rtcheck.has_value()) j["realtimeCheck"] = *a.rtcheck; + + return j; + } + + nlohmann::json envLayer (const EnvProvider& env) + { + auto j = nlohmann::json::object(); + + auto setString = [&] (const char* name, const char* key) + { + if (auto v = env (name); v.isNotEmpty()) + j[key] = v.toStdString(); + }; + auto setFlag = [&] (const char* name, const char* key) + { + if (env (name).isNotEmpty()) + j[key] = true; + }; + + if (auto v = env ("STRICTNESS_LEVEL"); v.isNotEmpty()) j["strictnessLevel"] = v.getIntValue(); + if (auto v = env ("RANDOM_SEED"); v.isNotEmpty()) j["randomSeed"] = settings_serializer::parseRandomSeed (v); + if (auto v = env ("TIMEOUT_MS"); v.isNotEmpty()) j["timeoutMs"] = (std::int64_t) v.getLargeIntValue(); + if (auto v = env ("REPEAT"); v.isNotEmpty()) j["numRepeats"] = v.getIntValue(); + setFlag ("VERBOSE", "verbose"); + setFlag ("RANDOMISE", "randomiseTestOrder"); + setFlag ("SKIP_GUI_TESTS", "skipGuiTests"); + setString ("DATA_FILE", "dataFile"); + setString ("OUTPUT_DIR", "outputDir"); + setString ("OUTPUT_FILENAME", "outputFilename"); + if (auto v = env ("SAMPLE_RATES"); v.isNotEmpty()) j["sampleRates"] = settings_serializer::commaToDoubleArray (v); + if (auto v = env ("BLOCK_SIZES"); v.isNotEmpty()) j["blockSizes"] = settings_serializer::commaToIntArray (v); + setString ("RTCHECK", "realtimeCheck"); + + return j; + } + } + + //============================================================================== + juce::StringArray preprocess (const juce::String& commandLineIn) + { + if (commandLineIn.contains ("strictnessLevel")) + { + std::cout << "!!! WARNING:\n\t\"strictnessLevel\" is deprecated and will be removed in a future version.\n" + << "\tPlease use --strictness-level instead\n\n"; + } + + const auto commandLine = commandLineIn.replace ("strictnessLevel", "strictness-level") + .replace ("-NSDocumentRevisionsDebugMode YES", "") + .trim(); + + juce::StringArray args; + args.addTokens (commandLine, true); + args.trim(); + + for (auto& s : args) + s = s.unquoted(); + + // If only a plugin path is supplied as the last arg, add an implicit + // --validate option for it so the rest of the CLI works. + if (args.size() > 0) + { + const bool hasCommand = hasOption (args, "--validate") + || args.contains ("--help") || args.contains ("-h") + || args.contains ("--version") + || args.contains ("--run-tests") + || hasOption (args, "--config-base64"); + + if (! hasCommand && isPluginArgument (args[args.size() - 1])) + args.insert (args.size() - 1, "--validate"); + } + + return args; + } + + bool isCommandLine (const juce::StringArray& tokens) + { + return tokens.contains ("--help") || tokens.contains ("-h") + || tokens.contains ("--version") + || hasOption (tokens, "--validate") + || tokens.contains ("--run-tests") + || tokens.contains ("--strictness-help") + || hasOption (tokens, "--config-base64"); + } + + juce::String resolvePluginPath (const juce::String& raw) + { + // Resolve relative/home paths against the working directory. Absolute + // paths and bare component IDs (no '.' or '~') are left untouched. + if (raw.contains ("~") || raw.contains (".")) + return juce::File::getCurrentWorkingDirectory().getChildFile (raw).getFullPathName(); + + return raw; + } + + PluginvalSettings resolveSettings (const juce::StringArray& tokens, const EnvProvider& env) + { + // A base64 JSON handoff from the parent process is fully authoritative. + if (const auto b64 = valueForOption (tokens, "--config-base64"); b64.isNotEmpty()) + { + auto s = settings_serializer::fromJsonString (decodeBase64 (b64).toStdString()); + s.validatePath = resolvePluginPath (juce::String (s.validatePath)).toStdString(); + return s; + } + + auto configJson = nlohmann::json::object(); + + if (const auto configPath = valueForOption (tokens, "--config"); configPath.isNotEmpty()) + configJson = nlohmann::json::parse (juce::File (configPath).loadFileAsString().toStdString()); + + const auto envJson = envLayer (env); + const auto cliJson = parseCliLayer (tokens); + + // Precedence (later wins): defaults < config < env < CLI. + auto merged = nlohmann::json::object(); + merged.merge_patch (configJson); + merged.merge_patch (envJson); + merged.merge_patch (cliJson); + + auto s = merged.get(); + s.validatePath = resolvePluginPath (juce::String (s.validatePath)).toStdString(); + return s; + } + + PluginvalSettings parse (const juce::String& commandLine, const EnvProvider& env) + { + return resolveSettings (preprocess (commandLine), env); + } + + //============================================================================== + juce::StringArray createChildProcessCommandLine (const juce::String& fileOrID, const PluginTests::Options& options) + { + const auto settings = PluginvalSettings::fromPluginTestOptions (options, fileOrID); + const auto jsonString = settings_serializer::toJson (settings).dump(); + const auto b64 = juce::Base64::toBase64 (juce::String (jsonString)); + + juce::StringArray args (juce::File::getSpecialLocation (juce::File::currentExecutableFile).getFullPathName()); + args.addArray ({ "--config-base64", b64, "--validate", fileOrID }); + return args; + } + + //============================================================================== + void printHelp (const juce::String& exeName) + { + std::vector storage { exeName.toStdString(), "--help" }; + std::vector views (storage.begin(), storage.end()); + + // magic_args prints the auto-generated usage to stdout and returns HelpRequested. + (void) magic_args::parse (std::span (views), makeProgramInfo()); + + std::cout << "\n" << getTrailerText() << std::endl; + } +} diff --git a/Source/SettingsParser.h b/Source/SettingsParser.h new file mode 100644 index 0000000..302457b --- /dev/null +++ b/Source/SettingsParser.h @@ -0,0 +1,72 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#pragma once + +#include "PluginvalSettings.h" +#include "PluginTests.h" + +#include + +/** + The command-line -> settings pipeline. + + Raw command line is preprocessed into argv tokens, then each input layer + (config file, environment, CLI) becomes a sparse JSON object. The layers are + merged (CLI wins last) and deserialised into a single PluginvalSettings. +*/ +namespace settings_parser +{ + /** Returns the value of an environment variable, or "" if unset. */ + using EnvProvider = std::function; + juce::String systemEnv (const juce::String& name); + + //============================================================================== + /** Tokenises + preprocesses a raw command line: rewrites the deprecated + "strictnessLevel", strips the macOS "-NSDocumentRevisionsDebugMode YES" + flag, and inserts an implicit "--validate" when the last argument is a + bare plugin path. Returns argv tokens (without the program name). + */ + juce::StringArray preprocess (const juce::String& commandLine); + + /** True if the tokens contain a recognised command that triggers CLI mode. */ + bool isCommandLine (const juce::StringArray& tokens); + + //============================================================================== + /** Resolves the merged settings from preprocessed tokens + environment. */ + PluginvalSettings resolveSettings (const juce::StringArray& tokens, const EnvProvider& env = systemEnv); + + /** Convenience: preprocess + resolveSettings from a raw command line. */ + PluginvalSettings parse (const juce::String& commandLine, const EnvProvider& env = systemEnv); + + /** Resolves a relative/home plugin path against the working directory, + leaving absolute paths and bare component IDs untouched. + */ + juce::String resolvePluginPath (const juce::String& raw); + + //============================================================================== + /** Serialises options for the child validation process as a base64 JSON + handoff: { exe, --config-base64 , --validate }. + */ + juce::StringArray createChildProcessCommandLine (const juce::String& fileOrID, const PluginTests::Options&); + + //============================================================================== + /** Prints the auto-generated usage (magic_args) plus the environment-variable + and commands trailer to stdout. + */ + void printHelp (const juce::String& exeName); + + /** The "--version" output string. */ + juce::String getVersionString(); +} diff --git a/Source/SettingsSerializer.cpp b/Source/SettingsSerializer.cpp new file mode 100644 index 0000000..9eb62c4 --- /dev/null +++ b/Source/SettingsSerializer.cpp @@ -0,0 +1,88 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#include "SettingsSerializer.h" + +#include + +namespace settings_serializer +{ + nlohmann::json toJson (const PluginvalSettings& s) + { + return nlohmann::json (s); + } + + PluginvalSettings fromJsonString (const std::string& jsonString) + { + return nlohmann::json::parse (jsonString).get(); + } + + PluginvalSettings fromJsonFile (const juce::File& file) + { + return fromJsonString (file.loadFileAsString().toStdString()); + } + + //============================================================================== + std::int64_t parseRandomSeed (const juce::String& raw) + { + if (! raw.containsOnly ("x-0123456789acbdef")) + throw std::invalid_argument ("Invalid random seed argument!"); + + if (raw.startsWith ("0x")) + return raw.getHexValue64(); + + return raw.getLargeIntValue(); + } + + nlohmann::json commaToDoubleArray (const juce::String& raw) + { + auto out = nlohmann::json::array(); + + for (const auto& token : juce::StringArray::fromTokens (raw, ",", "\"")) + out.push_back (token.getDoubleValue()); + + return out; + } + + nlohmann::json commaToIntArray (const juce::String& raw) + { + auto out = nlohmann::json::array(); + + for (const auto& token : juce::StringArray::fromTokens (raw, ",", "\"")) + out.push_back (token.getIntValue()); + + return out; + } + + nlohmann::json disabledTestsToArray (const juce::String& raw) + { + auto out = nlohmann::json::array(); + + if (juce::File::isAbsolutePath (raw)) + { + juce::StringArray lines; + juce::File (raw).readLines (lines); + + for (const auto& line : lines) + out.push_back (line.toStdString()); + + return out; + } + + for (const auto& token : juce::StringArray::fromTokens (raw, ",", "")) + out.push_back (token.toStdString()); + + return out; + } +} diff --git a/Source/SettingsSerializer.h b/Source/SettingsSerializer.h new file mode 100644 index 0000000..845d298 --- /dev/null +++ b/Source/SettingsSerializer.h @@ -0,0 +1,56 @@ +/*============================================================================== + + Copyright 2018 by Tracktion Corporation. + For more information visit www.tracktion.com + + You may also use this code under the terms of the GPL v3 (see + www.gnu.org/licenses). + + pluginval IS PROVIDED "AS IS" WITHOUT ANY WARRANTY, AND ALL WARRANTIES, WHETHER + EXPRESSED OR IMPLIED, INCLUDING MERCHANTABILITY AND FITNESS FOR PURPOSE, ARE + DISCLAIMED. + + ==============================================================================*/ + +#pragma once + +#include "PluginvalSettings.h" +#include + +#include +#include + +/** + JSON (de)serialisation for PluginvalSettings plus the small value coercions + shared by the CLI and environment-variable layers (which arrive as raw + strings and must become typed JSON values). +*/ +namespace settings_serializer +{ + //============================================================================== + /** Full JSON representation of a settings set (all fields present). */ + nlohmann::json toJson (const PluginvalSettings&); + + /** Parses a complete settings set from a JSON string (missing keys -> defaults). */ + PluginvalSettings fromJsonString (const std::string&); + + /** Parses a complete settings set from a JSON file (missing keys -> defaults). */ + PluginvalSettings fromJsonFile (const juce::File&); + + //============================================================================== + /** Parses a random seed from "0x..." hex or a decimal integer, preserving the + historical character-set validation. Throws std::invalid_argument if invalid. + */ + std::int64_t parseRandomSeed (const juce::String& raw); + + /** Splits a comma-separated list into a JSON array of doubles. */ + nlohmann::json commaToDoubleArray (const juce::String& raw); + + /** Splits a comma-separated list into a JSON array of ints. */ + nlohmann::json commaToIntArray (const juce::String& raw); + + /** Resolves --disabled-tests: an absolute path is read line-by-line, otherwise + the value is treated as a comma-separated list. Returns a JSON string array. + */ + nlohmann::json disabledTestsToArray (const juce::String& raw); +} From 08222148ef803b37ff48c7d351a8929a20bb2a43 Mon Sep 17 00:00:00 2001 From: David Rowland Date: Sat, 6 Jun 2026 22:22:31 +0100 Subject: [PATCH 03/12] Regenerate CLI docs and update architecture notes - docs/Command line options.md regenerated from magic_args --help output - CLAUDE.md: document the JSON-merge settings pipeline, new source files, magic_args + nlohmann/json deps, C++23, and the macOS 13.3 deployment target Co-Authored-By: Claude Opus 4.8 --- CLAUDE.md | 37 +++++++++-- docs/Command line options.md | 117 ++++++++++++----------------------- 2 files changed, 73 insertions(+), 81 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index fcec688..7606c12 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -7,7 +7,7 @@ - **Version**: 1.0.4 (see `VERSION` file) - **License**: GPLv3 - **Framework**: Built on JUCE (v8.0.x) -- **Language**: C++20 +- **Language**: C++23 ### Key Features - Tests VST/VST2/VST3/AU/LV2/LADSPA plugins @@ -80,7 +80,10 @@ pluginval/ │ ├── MainComponent.cpp/h # GUI main window component │ ├── Validator.cpp/h # Core validation orchestration │ ├── PluginTests.cpp/h # Test framework and base classes -│ ├── CommandLine.cpp/h # CLI argument parsing +│ ├── CommandLine.cpp/h # Thin CLI adapter (delegates to SettingsParser) +│ ├── PluginvalSettings.h # Unified settings struct + JSON mapping + toPluginTestOptions() +│ ├── SettingsParser.cpp/h # CLI/env/config -> merged JSON -> settings; child-process handoff +│ ├── SettingsSerializer.cpp/h # JSON load/save + value coercions (comma lists, hex seed) │ ├── CrashHandler.cpp/h # Crash reporting utilities │ ├── TestUtilities.cpp/h # Helper functions for tests │ ├── RTCheck.h # Real-time safety checking macros @@ -122,7 +125,7 @@ pluginval/ ### Prerequisites - CMake 3.15+ -- C++20 compatible compiler +- C++23 compatible compiler - Git (for submodules) ### Building @@ -156,7 +159,7 @@ VST2_SDK_DIR=/path/to/vst2sdk cmake -B Builds/Debug . ``` ### Target Platforms -- **macOS**: 10.11+ (deployment target), supports Apple Silicon via universal binary +- **macOS**: 13.3+ (deployment target — required by std::format used in magic_args), supports Apple Silicon via universal binary - **Windows**: MSVC with static runtime linking - **Linux**: Ubuntu 22.04+, statically links libstdc++ @@ -188,6 +191,29 @@ VST2_SDK_DIR=/path/to/vst2sdk cmake -B Builds/Debug . - Auto-registers via static instance pattern - Defines requirements (thread, GUI needs) +### CLI Settings Pipeline + +Command-line parsing is a layered JSON-merge pipeline rather than a bespoke +parser. The flow (in `SettingsParser`): + +1. **preprocess** the raw command line — rewrite the deprecated `strictnessLevel`, + strip the macOS `-NSDocumentRevisionsDebugMode YES` flag, and insert an + implicit `--validate` when the last argument is a bare plugin path. +2. Build a **sparse `nlohmann::json` per layer**: `--config` file, environment + variables, and the CLI options (parsed with **magic_args** into a struct of + `std::optional` fields, then coerced — comma lists → arrays, hex/int seed → + number, etc. via `SettingsSerializer`). +3. **Merge** with `merge_patch` in precedence order (defaults < config < env < + CLI; **CLI wins**), then deserialise into `PluginvalSettings` + (`NLOHMANN_DEFINE_TYPE..._WITH_DEFAULT` fills missing keys from defaults). +4. `PluginvalSettings::toPluginTestOptions()` converts to the JUCE-flavoured + `PluginTests::Options` at the boundary. + +The child validation process receives a fully-resolved, **authoritative** +settings set via a base64-encoded JSON argument (`--config-base64`), avoiding +per-flag re-serialisation and command-line quoting hazards. `--help`/`--version` +are handled by magic_args (auto usage + an appended env-var trailer). + ### Test Framework Tests are self-registering. To find all tests, look for static instances: @@ -327,6 +353,7 @@ Basic usage: Key options: - `--validate [path]` - Validate plugin at path +- `--config [file.json]` - Load a full settings set from JSON (overridden by env vars and CLI options) - `--strictness-level [1-10]` - Test thoroughness (default: 5) - `--skip-gui-tests` - Skip GUI tests (for headless CI) - `--validate-in-process` - Don't use child process (for debugging) @@ -374,6 +401,8 @@ add_pluginval_tests(MyPluginTarget ### External - **JUCE** (v8.0.x) - Audio application framework (git submodule) - **magic_enum** (v0.9.7) - Enum reflection (fetched via CPM) +- **magic_args** (v0.2.1) - C++23 CLI argument parsing, header-only (fetched via CPM); requires macOS 13.3+ due to std::format +- **nlohmann/json** (3.12.0) - JSON settings layering/serialisation (fetched via CPM) - **rtcheck** (optional, macOS) - Real-time safety checking (fetched via CPM) - **VST3 SDK** (v3.7.x) - Steinberg VST3 SDK for embedded validator (fetched via CPM, optional) diff --git a/docs/Command line options.md b/docs/Command line options.md index 4af7339..3af2dc2 100644 --- a/docs/Command line options.md +++ b/docs/Command line options.md @@ -1,85 +1,48 @@ -Hello rtcheck! -//============================================================================== -pluginval -JUCE v8.0.10 - -Description: - Validate plugins to test compatibility with hosts and verify plugin API conformance - -Usage: - --version - Print pluginval version. - --validate [pathToPlugin] - Validates the plugin at the given path. - N.B. the "--validate" flag is optional if the path is the last argument. - This enables you to validate a plugin with simply "pluginval path_to_plugin". - - --sample-rates [list of comma separated sample rates] - If specified, sets the list of sample rates at which tests will be executed - (default=44100,48000,96000) - --block-sizes [list of comma separated block sizes] - If specified, sets the list of block sizes at which tests will be executed - (default=64,128,256,512,1024) - --random-seed [hex or int] - Sets the random seed to use for the tests. Useful for replicating test - environments. - --data-file [pathToFile] - If specified, sets a path to a data file which can be used by tests to - configure themselves. This can be useful for things like known audio output. - - --strictness-level [1-10] - Sets the strictness level to use. A minimum level of 5 (also the default) - is recomended for compatibility. - Higher levels include longer, more thorough tests such as fuzzing. - --timeout-ms [numMilliseconds] - Sets a timout which will stop validation with an error if no output from any - test has happened for this number of ms. - By default this is 30s but can be set to "-1" (must be quoted) to never timeout. - --rtcheck [empty, disabled, enabled or relaxed] - Turns on real-time saftey checks using rtcheck (macOS and Linux only). - relaxed mode doesn't run the checks for the first processing block as a lot of plugins - use this to allocate or initialise thread-locals (which can allocate) - - --repeat [num repeats] - If specified repeats the tests a given number of times. Note that this does - not delete and re-instantiate the plugin for each repeat. - --randomise - If specified, the tests are run in a random order per repeat. - - --skip-gui-tests - If specified, avoids tests that create GUI windows, which can cause problems - on headless CI systems. - --disabled-tests [pathToFile] - If specified, sets a path to a file that should have the names of disabled - tests on each row. - --vst3validator [pathToValidator] - If specified, this will run the VST3 validator as part of the test process. - - --output-dir [pathToDir] - If specified, sets a directory to store the log files. This can be useful - for continuous integration. - --output-filename [filename] - If specified, sets a filename for the log files (within 'output-dir' or - (lacking that) the current directory. - By default, the name is constructed from the plugin metainformation - --verbose - If specified, outputs additional logging information. It can be useful to - turn this off when building with CI to avoid huge log files. - -Exit code: +Usage: pluginval [OPTIONS...] +Validate plugins to test compatibility with hosts and verify plugin API conformance + +Examples: + + pluginval --strictness-level 5 --validate path/to/plugin + pluginval path/to/plugin + pluginval --config settings.json --validate path/to/plugin + +Options: + + --validate=VALUE Validates the plugin at the given path (or AU id). Optional if the path is the last argument. + --strictness-level=VALUE + Strictness level 1-10 (default 5, minimum 5 recommended). + --random-seed=VALUE Random seed (hex 0x.. or int) for replicable test runs. + --timeout-ms=VALUE Test timeout in ms (default 30000, -1 to never timeout). + --repeat=VALUE Number of times to repeat the tests. + --randomise Run the tests in a random order per repeat. + --verbose Output additional logging information. + --skip-gui-tests Avoid tests that create GUI windows (for headless CI). + --sample-rates=VALUE Comma-separated sample rates (default 44100,48000,96000). + --block-sizes=VALUE Comma-separated block sizes (default 64,128,256,512,1024). + --data-file=VALUE Path to a data file tests can use to configure themselves. + --output-dir=VALUE Directory in which to write the log files. + --output-filename=VALUE Filename to write logs into. + --disabled-tests=VALUE Comma-separated test names, or a path to a file listing them. + --rtcheck=VALUE Real-time safety checks: disabled, enabled or relaxed. + --config=VALUE Path to a JSON file of settings (overridden by env vars and CLI options). + + -?, --help show this message + --version print program version + +Other commands: + --version Print the pluginval version. + --run-tests Run the internal unit tests. + --strictness-help [level] List all tests that run at the given strictness level. + +Exit code: 0 if all tests complete successfully 1 if there are any errors Additionally, you can specify any of the command line options as environment variables by removing prefix dashes, converting internal dashes to underscores -and capitalising all letters, a.g. +and capitalising all letters, e.g. "--skip-gui-tests" > "SKIP_GUI_TESTS=1" "--timeout-ms 30000" > "TIMEOUT_MS=30000" -Specifying specific command-line options will override any environment variables -set for that option. - - pluginval --version Prints the current version number - pluginval --help|-h Prints the list of commands - pluginval --validate [pathToPlugin] Validates the file (or IDs for AUs). - pluginval --run-tests Runs the internal unit tests. +Precedence is command-line options > environment variables > --config file. From 18c7a0b56d9f55ccd47883676473cafce863f685 Mon Sep 17 00:00:00 2001 From: David Rowland Date: Sat, 6 Jun 2026 23:15:42 +0100 Subject: [PATCH 04/12] Swap magic_args for CLI11: bind options directly to the settings struct Addresses review feedback that the magic_args version duplicated the settings into a parallel PluginvalCliArgs struct plus manual CLI->JSON and ENV->JSON bridges (5-6 edit sites per new option), and forced cxx_std_23 + a macOS 13.3 deployment target (std::format). CLI11 binds each option directly to a PluginvalSettings member, with the environment variable on the same line via ->envname(). Adding an option is now three edits: a struct member, a nlohmann macro entry, and one add_option(...) line. No parallel struct, no cliArgsToJson, no envLayer. - SettingsParser: rewritten on CLI11; comma lists via ->delimiter(','), enum via CheckedTransformer, hex/int seed via a small callback. --config seeds the struct before parse so precedence stays defaults CLI11 2.6.2; revert cxx_std_23 -> cxx_std_20 and CMAKE_OSX_DEPLOYMENT_TARGET 13.3 -> 10.11. - Docs/CLAUDE.md updated. Verified: full build (validator ON, C++20, macOS 10.11), --run-tests pass, real --validate SUCCESS, --config precedence covered by tests. Co-Authored-By: Claude Opus 4.8 --- CLAUDE.md | 32 ++-- CMakeLists.txt | 11 +- Source/CommandLine.cpp | 30 ++-- Source/CommandLineTests.cpp | 147 ++++++++++-------- Source/SettingsParser.cpp | 280 ++++++++++++---------------------- Source/SettingsParser.h | 42 +++-- Source/SettingsSerializer.cpp | 24 +-- Source/SettingsSerializer.h | 16 +- docs/Command line options.md | 96 ++++++------ 9 files changed, 297 insertions(+), 381 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 7606c12..3e00941 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -7,7 +7,7 @@ - **Version**: 1.0.4 (see `VERSION` file) - **License**: GPLv3 - **Framework**: Built on JUCE (v8.0.x) -- **Language**: C++23 +- **Language**: C++20 ### Key Features - Tests VST/VST2/VST3/AU/LV2/LADSPA plugins @@ -125,7 +125,7 @@ pluginval/ ### Prerequisites - CMake 3.15+ -- C++23 compatible compiler +- C++20 compatible compiler - Git (for submodules) ### Building @@ -159,7 +159,7 @@ VST2_SDK_DIR=/path/to/vst2sdk cmake -B Builds/Debug . ``` ### Target Platforms -- **macOS**: 13.3+ (deployment target — required by std::format used in magic_args), supports Apple Silicon via universal binary +- **macOS**: 10.11+ (deployment target), supports Apple Silicon via universal binary - **Windows**: MSVC with static runtime linking - **Linux**: Ubuntu 22.04+, statically links libstdc++ @@ -193,26 +193,30 @@ VST2_SDK_DIR=/path/to/vst2sdk cmake -B Builds/Debug . ### CLI Settings Pipeline -Command-line parsing is a layered JSON-merge pipeline rather than a bespoke -parser. The flow (in `SettingsParser`): +Command-line parsing centres on one plain settings struct (`PluginvalSettings`) +that CLI11 binds to directly. The flow (in `SettingsParser`): 1. **preprocess** the raw command line — rewrite the deprecated `strictnessLevel`, strip the macOS `-NSDocumentRevisionsDebugMode YES` flag, and insert an implicit `--validate` when the last argument is a bare plugin path. -2. Build a **sparse `nlohmann::json` per layer**: `--config` file, environment - variables, and the CLI options (parsed with **magic_args** into a struct of - `std::optional` fields, then coerced — comma lists → arrays, hex/int seed → - number, etc. via `SettingsSerializer`). -3. **Merge** with `merge_patch` in precedence order (defaults < config < env < - CLI; **CLI wins**), then deserialise into `PluginvalSettings` - (`NLOHMANN_DEFINE_TYPE..._WITH_DEFAULT` fills missing keys from defaults). +2. If `--config ` is present, **seed** the struct from that JSON first. +3. **CLI11** binds each `add_option`/`add_flag` straight to a `PluginvalSettings` + member, with the environment variable on the same line via `->envname()`. + Because CLI11 only overwrites a member when its flag/env was actually provided, + precedence is **defaults < config < env < CLI** (CLI wins) with no manual + layering. Comma lists use `->delimiter(',')`, the enum uses a + `CheckedTransformer`, and the hex/int seed is a small callback. 4. `PluginvalSettings::toPluginTestOptions()` converts to the JUCE-flavoured `PluginTests::Options` at the boundary. +Adding a new option is three edits: a struct member, an entry in the nlohmann +macro list, and one `add_option(...)` line. `SettingsSerializer` handles JSON +load/save plus the two remaining conversions (hex seed, disabled-tests file). + The child validation process receives a fully-resolved, **authoritative** settings set via a base64-encoded JSON argument (`--config-base64`), avoiding per-flag re-serialisation and command-line quoting hazards. `--help`/`--version` -are handled by magic_args (auto usage + an appended env-var trailer). +are handled by CLI11 (auto usage + a footer with the env-var/commands notes). ### Test Framework @@ -401,7 +405,7 @@ add_pluginval_tests(MyPluginTarget ### External - **JUCE** (v8.0.x) - Audio application framework (git submodule) - **magic_enum** (v0.9.7) - Enum reflection (fetched via CPM) -- **magic_args** (v0.2.1) - C++23 CLI argument parsing, header-only (fetched via CPM); requires macOS 13.3+ due to std::format +- **CLI11** (v2.6.2) - CLI argument parsing, header-only (fetched via CPM) - **nlohmann/json** (3.12.0) - JSON settings layering/serialisation (fetched via CPM) - **rtcheck** (optional, macOS) - Real-time safety checking (fetched via CPM) - **VST3 SDK** (v3.7.x) - Steinberg VST3 SDK for embedded validator (fetched via CPM, optional) diff --git a/CMakeLists.txt b/CMakeLists.txt index 932c81a..180d6ef 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,9 +6,8 @@ project(pluginval VERSION ${CURRENT_VERSION}) # Just compliing pluginval if (pluginval_IS_TOP_LEVEL) if (APPLE) - # Target OS versions down to 13.3. std::format (used by magic_args) pulls in - # libc++'s floating-point formatter, whose to_chars is only available on 13.3+. - set (CMAKE_OSX_DEPLOYMENT_TARGET "13.3" CACHE INTERNAL "") + # Target OS versions down to 10.11 + set (CMAKE_OSX_DEPLOYMENT_TARGET "10.11" CACHE INTERNAL "") # Uncomment to produce a universal binary # set(CMAKE_OSX_ARCHITECTURES arm64 x86_64) @@ -62,7 +61,7 @@ option(JUCE_ENABLE_MODULE_SOURCE_GROUPS "Enable Module Source Groups" ON) include(cmake/CPM.cmake) CPMAddPackage("gh:Neargye/magic_enum#v0.9.7") -CPMAddPackage("gh:fredemmott/magic_args#v0.2.1") +CPMAddPackage("gh:CLIUtils/CLI11@2.6.2") CPMAddPackage("gh:nlohmann/json@3.12.0") if(PLUGINVAL_ENABLE_RTCHECK) @@ -112,7 +111,7 @@ juce_add_gui_app(pluginval juce_generate_juce_header(pluginval) -target_compile_features(pluginval PRIVATE cxx_std_23) +target_compile_features(pluginval PRIVATE cxx_std_20) set_target_properties(pluginval PROPERTIES C_VISIBILITY_PRESET hidden @@ -185,7 +184,7 @@ target_link_libraries(pluginval PRIVATE juce::juce_audio_utils juce::juce_recommended_warning_flags magic_enum - magic_args::magic_args + CLI11::CLI11 nlohmann_json::nlohmann_json) if (${CMAKE_SYSTEM_NAME} MATCHES "Linux") diff --git a/Source/CommandLine.cpp b/Source/CommandLine.cpp index 900bad0..e561d39 100644 --- a/Source/CommandLine.cpp +++ b/Source/CommandLine.cpp @@ -196,20 +196,6 @@ void performCommandLine (CommandLineValidator& validator, const juce::String& co auto& app = *juce::JUCEApplication::getInstance(); const auto tokens = settings_parser::preprocess (commandLine); - if (tokens.contains ("--help") || tokens.contains ("-h")) - { - settings_parser::printHelp (app.getApplicationName()); - app.quit(); - return; - } - - if (tokens.contains ("--version")) - { - std::cout << settings_parser::getVersionString() << std::endl; - app.quit(); - return; - } - if (tokens.contains ("--run-tests")) { runUnitTests(); @@ -230,10 +216,20 @@ void performCommandLine (CommandLineValidator& validator, const juce::String& co return; } - // Otherwise this is a validation run (explicit or implicit --validate) + // Otherwise this is a validation run (explicit or implicit --validate). + // CLI11 handles --help/--version and parse errors. try { - auto [fileOrID, options] = parseCommandLine (commandLine); + const auto result = settings_parser::parseTokens (tokens); + + if (result.handled) + { + app.setApplicationReturnValue (result.exitCode); + app.quit(); + return; + } + + const auto fileOrID = juce::String (result.settings.validatePath); if (fileOrID.isEmpty()) { @@ -242,7 +238,7 @@ void performCommandLine (CommandLineValidator& validator, const juce::String& co } // --validate runs async so will quit itself when done - validator.validate (fileOrID, options); + validator.validate (fileOrID, result.settings.toPluginTestOptions()); } catch (const std::exception& e) { diff --git a/Source/CommandLineTests.cpp b/Source/CommandLineTests.cpp index 9d2ba44..9d22345 100644 --- a/Source/CommandLineTests.cpp +++ b/Source/CommandLineTests.cpp @@ -12,6 +12,7 @@ ==============================================================================*/ +#include struct CommandLineTests : public juce::UnitTest { @@ -20,22 +21,49 @@ struct CommandLineTests : public juce::UnitTest { } - // A deterministic, empty environment so tests don't pick up the host's env vars. - static settings_parser::EnvProvider emptyEnv() + static constexpr const char* knownEnvVars[] = { + "STRICTNESS_LEVEL", "RANDOM_SEED", "TIMEOUT_MS", "VERBOSE", "REPEAT", + "RANDOMISE", "SKIP_GUI_TESTS", "DATA_FILE", "OUTPUT_DIR", "OUTPUT_FILENAME", + "SAMPLE_RATES", "BLOCK_SIZES", "RTCHECK" + }; + + static void setEnv (const char* name, const char* value) + { + #if JUCE_WINDOWS + _putenv_s (name, value); + #else + ::setenv (name, value, 1); + #endif + } + + static void unsetEnv (const char* name) + { + #if JUCE_WINDOWS + _putenv_s (name, ""); + #else + ::unsetenv (name); + #endif + } + + static void clearKnownEnv() { - return [] (const juce::String&) { return juce::String(); }; + for (auto* n : knownEnvVars) + unsetEnv (n); } - static PluginvalSettings parse (const juce::String& cmd, settings_parser::EnvProvider env) + static PluginvalSettings parse (const juce::String& cmd) { - return settings_parser::resolveSettings (settings_parser::preprocess (cmd), env); + return settings_parser::parse (cmd); } void runTest() override { + // Start from a clean environment so host env vars don't affect the deterministic tests. + clearKnownEnv(); + beginTest ("Command line defaults"); { - const auto opts = parse ("", emptyEnv()).toPluginTestOptions(); + const auto opts = parse ("").toPluginTestOptions(); expectEquals (opts.strictnessLevel, 5); expectEquals (opts.randomSeed, (juce::int64) 0); expectEquals (opts.timeoutMs, (juce::int64) 30000); @@ -53,8 +81,7 @@ struct CommandLineTests : public juce::UnitTest beginTest ("Command line parser"); { const auto settings = parse ("--strictness-level 7 --random-seed 1234 --timeout-ms 20000 --repeat 11 " - "--data-file /path/to/file --output-dir /path/to/dir --validate /path/to/plugin", - emptyEnv()); + "--data-file /path/to/file --output-dir /path/to/dir --validate /path/to/plugin"); const auto opts = settings.toPluginTestOptions(); expectEquals (opts.strictnessLevel, 7); expectEquals (opts.randomSeed, (juce::int64) 1234); @@ -65,31 +92,36 @@ struct CommandLineTests : public juce::UnitTest expectEquals (juce::String (settings.validatePath), juce::String ("/path/to/plugin")); } + beginTest ("Negative timeout"); + { + expectEquals (parse ("--timeout-ms -1 --validate x").timeoutMs, (juce::int64) -1); + } + beginTest ("Command line random (hex and int)"); { - expectEquals (parse ("--random-seed 0x7f2da1 --validate x", emptyEnv()).randomSeed, (juce::int64) 8334753); - expectEquals (parse ("--random-seed 0x692bc1f --validate x", emptyEnv()).randomSeed, (juce::int64) 110279711); - expectEquals (parse ("--random-seed 1234 --validate x", emptyEnv()).randomSeed, (juce::int64) 1234); + expectEquals (parse ("--random-seed 0x7f2da1 --validate x").randomSeed, (juce::int64) 8334753); + expectEquals (parse ("--random-seed 0x692bc1f --validate x").randomSeed, (juce::int64) 110279711); + expectEquals (parse ("--random-seed 1234 --validate x").randomSeed, (juce::int64) 1234); } beginTest ("Comma-separated lists"); { - const auto opts = parse ("--sample-rates 22050,44100 --block-sizes 32,64,128 --validate x", emptyEnv()).toPluginTestOptions(); + const auto opts = parse ("--sample-rates 22050,44100 --block-sizes 32,64,128 --validate x").toPluginTestOptions(); expect (opts.sampleRates == std::vector ({ 22050.0, 44100.0 })); expect (opts.blockSizes == std::vector ({ 32, 64, 128 })); } beginTest ("rtcheck enum parsing"); { - expect (parse ("--rtcheck relaxed --validate x", emptyEnv()).realtimeCheck == RealtimeCheck::relaxed); - expect (parse ("--rtcheck enabled --validate x", emptyEnv()).realtimeCheck == RealtimeCheck::enabled); - expect (parse ("--validate x", emptyEnv()).realtimeCheck == RealtimeCheck::disabled); + expect (parse ("--rtcheck relaxed --validate x").realtimeCheck == RealtimeCheck::relaxed); + expect (parse ("--rtcheck enabled --validate x").realtimeCheck == RealtimeCheck::enabled); + expect (parse ("--validate x").realtimeCheck == RealtimeCheck::disabled); } beginTest ("Handles an absolute path to the plugin"); { const auto homeDir = juce::File::getSpecialLocation (juce::File::userHomeDirectory).getFullPathName(); - expectEquals (juce::String (parse ("--validate " + homeDir + "/path/to/MyPlugin", emptyEnv()).validatePath), + expectEquals (juce::String (parse ("--validate " + homeDir + "/path/to/MyPlugin").validatePath), homeDir + "/path/to/MyPlugin"); } @@ -97,21 +129,21 @@ struct CommandLineTests : public juce::UnitTest { const auto homeDir = juce::File::getSpecialLocation (juce::File::userHomeDirectory).getFullPathName(); const auto pathToQuote = homeDir + "/path/to/MyPlugin"; - expectEquals (juce::String (parse ("--validate " + pathToQuote.quoted(), emptyEnv()).validatePath), + expectEquals (juce::String (parse ("--validate " + pathToQuote.quoted()).validatePath), homeDir + "/path/to/MyPlugin"); } beginTest ("Handles a relative path"); { const auto currentDir = juce::File::getCurrentWorkingDirectory(); - expectEquals (juce::String (parse ("--validate MyPlugin.vst3", emptyEnv()).validatePath), + expectEquals (juce::String (parse ("--validate MyPlugin.vst3").validatePath), currentDir.getChildFile ("MyPlugin.vst3").getFullPathName()); } beginTest ("Handles a quoted relative path with spaces to the plugin"); { const auto currentDir = juce::File::getCurrentWorkingDirectory(); - expectEquals (juce::String (parse (R"(--validate "My Plugin.vst3")", emptyEnv()).validatePath), + expectEquals (juce::String (parse (R"(--validate "My Plugin.vst3")").validatePath), currentDir.getChildFile ("My Plugin.vst3").getFullPathName()); } @@ -119,20 +151,20 @@ struct CommandLineTests : public juce::UnitTest beginTest ("Handles a relative path with ./ to the plugin"); { const auto currentDir = juce::File::getCurrentWorkingDirectory().getFullPathName(); - expectEquals (juce::String (parse ("--validate ./path/to/MyPlugin", emptyEnv()).validatePath), + expectEquals (juce::String (parse ("--validate ./path/to/MyPlugin").validatePath), currentDir + "/path/to/MyPlugin"); } beginTest ("Handles a home directory relative path to the plugin"); { - expectEquals (juce::String (parse ("--validate ~/path/to/MyPlugin", emptyEnv()).validatePath), + expectEquals (juce::String (parse ("--validate ~/path/to/MyPlugin").validatePath), juce::File::getSpecialLocation (juce::File::userHomeDirectory).getFullPathName() + "/path/to/MyPlugin"); } beginTest ("Handles quoted strings, spaces, and home directory relative path to the plugin"); { const auto cmd = R"(--data-file "~/path/to/My File" --output-dir "~/path/to/My Directory" --validate "~/path/to/My Plugin")"; - expectEquals (juce::String (parse (cmd, emptyEnv()).validatePath), + expectEquals (juce::String (parse (cmd).validatePath), juce::File::getSpecialLocation (juce::File::userHomeDirectory).getFullPathName() + "/path/to/My Plugin"); } #endif @@ -140,19 +172,19 @@ struct CommandLineTests : public juce::UnitTest beginTest ("Implicit validate with a relative path"); { const auto currentDir = juce::File::getCurrentWorkingDirectory(); - expectEquals (juce::String (parse ("MyPlugin.vst3", emptyEnv()).validatePath), + expectEquals (juce::String (parse ("MyPlugin.vst3").validatePath), currentDir.getChildFile ("MyPlugin.vst3").getFullPathName()); } beginTest ("Doesn't alter component IDs"); { - expectEquals (juce::String (parse ("--validate MyPluginID", emptyEnv()).validatePath), juce::String ("MyPluginID")); + expectEquals (juce::String (parse ("--validate MyPluginID").validatePath), juce::String ("MyPluginID")); } beginTest ("Allows for other options after explicit --validate"); { const auto currentDir = juce::File::getCurrentWorkingDirectory(); - const auto settings = parse ("--validate MyPlugin.vst3 --randomise", emptyEnv()); + const auto settings = parse ("--validate MyPlugin.vst3 --randomise"); expectEquals (juce::String (settings.validatePath), currentDir.getChildFile ("MyPlugin.vst3").getFullPathName()); expect (settings.randomiseTestOrder); } @@ -169,28 +201,23 @@ struct CommandLineTests : public juce::UnitTest beginTest ("Environment variables"); { - std::map envVars { - { "STRICTNESS_LEVEL", "7" }, - { "RANDOM_SEED", "1234" }, - { "TIMEOUT_MS", "20000" }, - { "VERBOSE", "1" }, - { "REPEAT", "11" }, - { "RANDOMISE", "1" }, - { "SKIP_GUI_TESTS", "1" }, - { "DATA_FILE", "/path/to/file" }, - { "OUTPUT_DIR", "/path/to/dir" }, - { "SAMPLE_RATES", "22050,44100" }, - { "BLOCK_SIZES", "32,64" }, - { "RTCHECK", "relaxed" }, - }; - - settings_parser::EnvProvider env = [envVars] (const juce::String& n) - { - const auto it = envVars.find (n); - return it != envVars.end() ? it->second : juce::String(); - }; + setEnv ("STRICTNESS_LEVEL", "7"); + setEnv ("RANDOM_SEED", "1234"); + setEnv ("TIMEOUT_MS", "20000"); + setEnv ("VERBOSE", "1"); + setEnv ("REPEAT", "11"); + setEnv ("RANDOMISE", "1"); + setEnv ("SKIP_GUI_TESTS", "1"); + setEnv ("DATA_FILE", "/path/to/file"); + setEnv ("OUTPUT_DIR", "/path/to/dir"); + setEnv ("SAMPLE_RATES", "22050,44100"); + setEnv ("BLOCK_SIZES", "32,64"); + setEnv ("RTCHECK", "relaxed"); + + const auto opts = parse ("--validate x").toPluginTestOptions(); + + clearKnownEnv(); - const auto opts = parse ("--validate x", env).toPluginTestOptions(); expectEquals (opts.strictnessLevel, 7); expectEquals (opts.randomSeed, (juce::int64) 1234); expectEquals (opts.timeoutMs, (juce::int64) 20000); @@ -205,13 +232,13 @@ struct CommandLineTests : public juce::UnitTest beginTest ("Command line overrides environment variables"); { - settings_parser::EnvProvider env = [] (const juce::String& n) - { - return n == "STRICTNESS_LEVEL" ? juce::String ("3") : juce::String(); - }; + setEnv ("STRICTNESS_LEVEL", "3"); + const auto envOnly = parse ("--validate x").strictnessLevel; + const auto cliWins = parse ("--strictness-level 9 --validate x").strictnessLevel; + clearKnownEnv(); - expectEquals (parse ("--validate x", env).strictnessLevel, 3); // env only - expectEquals (parse ("--strictness-level 9 --validate x", env).strictnessLevel, 9); // CLI wins + expectEquals (envOnly, 3); // env only + expectEquals (cliWins, 9); // CLI wins } beginTest ("Config file and precedence (CLI > env > config > defaults)"); @@ -222,7 +249,7 @@ struct CommandLineTests : public juce::UnitTest // config alone { - const auto s = parse (cfg + " --validate x", emptyEnv()); + const auto s = parse (cfg + " --validate x"); expectEquals (s.strictnessLevel, 2); expectEquals (s.timeoutMs, (juce::int64) 12345); expectEquals (s.numRepeats, 4); @@ -230,20 +257,18 @@ struct CommandLineTests : public juce::UnitTest // env overrides config { - settings_parser::EnvProvider env = [] (const juce::String& n) - { return n == "STRICTNESS_LEVEL" ? juce::String ("6") : juce::String(); }; - - const auto s = parse (cfg + " --validate x", env); + setEnv ("STRICTNESS_LEVEL", "6"); + const auto s = parse (cfg + " --validate x"); + clearKnownEnv(); expectEquals (s.strictnessLevel, 6); // env beats config expectEquals (s.timeoutMs, (juce::int64) 12345); // still from config } // CLI overrides env and config { - settings_parser::EnvProvider env = [] (const juce::String& n) - { return n == "STRICTNESS_LEVEL" ? juce::String ("6") : juce::String(); }; - - const auto s = parse (cfg + " --strictness-level 9 --validate x", env); + setEnv ("STRICTNESS_LEVEL", "6"); + const auto s = parse (cfg + " --strictness-level 9 --validate x"); + clearKnownEnv(); expectEquals (s.strictnessLevel, 9); expectEquals (s.timeoutMs, (juce::int64) 12345); } diff --git a/Source/SettingsParser.cpp b/Source/SettingsParser.cpp index e08041f..3c2bcb3 100644 --- a/Source/SettingsParser.cpp +++ b/Source/SettingsParser.cpp @@ -15,101 +15,15 @@ #include "SettingsParser.h" #include "SettingsSerializer.h" -#include +#include #include -#include +#include #include -#include #include namespace settings_parser { - //============================================================================== - // The magic_args view of the flat command-line options. Everything is parsed - // as a raw optional/flag; typed coercion happens when building the JSON layer. - struct PluginvalCliArgs - { - magic_args::option> validate - { .mName = "validate", .mHelp = "Validates the plugin at the given path (or AU id). Optional if the path is the last argument." }; - magic_args::option> strictnessLevel - { .mName = "strictness-level", .mHelp = "Strictness level 1-10 (default 5, minimum 5 recommended)." }; - magic_args::option> randomSeed - { .mName = "random-seed", .mHelp = "Random seed (hex 0x.. or int) for replicable test runs." }; - magic_args::option> timeoutMs - { .mName = "timeout-ms", .mHelp = "Test timeout in ms (default 30000, -1 to never timeout)." }; - magic_args::option> repeat - { .mName = "repeat", .mHelp = "Number of times to repeat the tests." }; - magic_args::flag randomise - { .mName = "randomise", .mHelp = "Run the tests in a random order per repeat." }; - magic_args::flag verbose - { .mName = "verbose", .mHelp = "Output additional logging information." }; - magic_args::flag skipGuiTests - { .mName = "skip-gui-tests", .mHelp = "Avoid tests that create GUI windows (for headless CI)." }; - magic_args::option> sampleRates - { .mName = "sample-rates", .mHelp = "Comma-separated sample rates (default 44100,48000,96000)." }; - magic_args::option> blockSizes - { .mName = "block-sizes", .mHelp = "Comma-separated block sizes (default 64,128,256,512,1024)." }; - magic_args::option> dataFile - { .mName = "data-file", .mHelp = "Path to a data file tests can use to configure themselves." }; - magic_args::option> outputDir - { .mName = "output-dir", .mHelp = "Directory in which to write the log files." }; - magic_args::option> outputFilename - { .mName = "output-filename", .mHelp = "Filename to write logs into." }; - magic_args::option> disabledTests - { .mName = "disabled-tests", .mHelp = "Comma-separated test names, or a path to a file listing them." }; - magic_args::option> rtcheck - { .mName = "rtcheck", .mHelp = "Real-time safety checks: disabled, enabled or relaxed." }; - magic_args::option> config - { .mName = "config", .mHelp = "Path to a JSON file of settings (overridden by env vars and CLI options)." }; - }; - - //============================================================================== - static magic_args::program_info makeProgramInfo() - { - return { - .mDescription = "Validate plugins to test compatibility with hosts and verify plugin API conformance", - .mVersion = getVersionString().toStdString(), - .mExamples = { - "pluginval --strictness-level 5 --validate path/to/plugin", - "pluginval path/to/plugin", - "pluginval --config settings.json --validate path/to/plugin", - }, - }; - } - - static juce::String getTrailerText() - { - return juce::String ( -R"(Other commands: - --version Print the pluginval version. - --run-tests Run the internal unit tests. - --strictness-help [level] List all tests that run at the given strictness level. - -Exit code: - 0 if all tests complete successfully - 1 if there are any errors - -Additionally, you can specify any of the command line options as environment -variables by removing prefix dashes, converting internal dashes to underscores -and capitalising all letters, e.g. - "--skip-gui-tests" > "SKIP_GUI_TESTS=1" - "--timeout-ms 30000" > "TIMEOUT_MS=30000" -Precedence is command-line options > environment variables > --config file. -)"); - } - - //============================================================================== - juce::String systemEnv (const juce::String& name) - { - return juce::SystemStats::getEnvironmentVariable (name, {}); - } - - juce::String getVersionString() - { - return juce::String ("pluginval") + " - " + VERSION; - } - //============================================================================== namespace { @@ -172,78 +86,29 @@ Precedence is command-line options > environment variables > --config file. return mos.toString(); } - // Builds the sparse CLI JSON layer by parsing tokens with magic_args. - nlohmann::json parseCliLayer (const juce::StringArray& tokens) + juce::String getFooterText() { - std::vector storage; - storage.reserve ((size_t) tokens.size() + 1); - storage.emplace_back ("pluginval"); - - for (const auto& t : tokens) - storage.push_back (t.toStdString()); - - std::vector views (storage.begin(), storage.end()); - - const auto parsed = magic_args::parse (std::span (views), - makeProgramInfo()); - - auto j = nlohmann::json::object(); - - if (! parsed) // HelpRequested / VersionRequested / parse error: contribute nothing - return j; - - const auto& a = *parsed; + return juce::String ( +R"(Other commands: + --run-tests Run the internal unit tests. + --strictness-help [level] List all tests that run at the given strictness level. - if (a.validate.has_value()) j["validatePath"] = *a.validate; - if (a.strictnessLevel.has_value()) j["strictnessLevel"] = *a.strictnessLevel; - if (a.randomSeed.has_value()) j["randomSeed"] = settings_serializer::parseRandomSeed (juce::String (*a.randomSeed)); - if (a.timeoutMs.has_value()) j["timeoutMs"] = *a.timeoutMs; - if (a.repeat.has_value()) j["numRepeats"] = *a.repeat; - if (a.verbose) j["verbose"] = true; - if (a.randomise) j["randomiseTestOrder"] = true; - if (a.skipGuiTests) j["skipGuiTests"] = true; - if (a.sampleRates.has_value()) j["sampleRates"] = settings_serializer::commaToDoubleArray (juce::String (*a.sampleRates)); - if (a.blockSizes.has_value()) j["blockSizes"] = settings_serializer::commaToIntArray (juce::String (*a.blockSizes)); - if (a.dataFile.has_value()) j["dataFile"] = *a.dataFile; - if (a.outputDir.has_value()) j["outputDir"] = *a.outputDir; - if (a.outputFilename.has_value()) j["outputFilename"] = *a.outputFilename; - if (a.disabledTests.has_value()) j["disabledTests"] = settings_serializer::disabledTestsToArray (juce::String (*a.disabledTests)); - if (a.rtcheck.has_value()) j["realtimeCheck"] = *a.rtcheck; +Exit code: + 0 if all tests complete successfully + 1 if there are any errors - return j; +You can also specify any option as an environment variable by removing the prefix +dashes, converting internal dashes to underscores and capitalising, e.g. + "--skip-gui-tests" -> "SKIP_GUI_TESTS=1" + "--timeout-ms 30000" -> "TIMEOUT_MS=30000" +Precedence: command-line options > environment variables > --config file.)"); } + } - nlohmann::json envLayer (const EnvProvider& env) - { - auto j = nlohmann::json::object(); - - auto setString = [&] (const char* name, const char* key) - { - if (auto v = env (name); v.isNotEmpty()) - j[key] = v.toStdString(); - }; - auto setFlag = [&] (const char* name, const char* key) - { - if (env (name).isNotEmpty()) - j[key] = true; - }; - - if (auto v = env ("STRICTNESS_LEVEL"); v.isNotEmpty()) j["strictnessLevel"] = v.getIntValue(); - if (auto v = env ("RANDOM_SEED"); v.isNotEmpty()) j["randomSeed"] = settings_serializer::parseRandomSeed (v); - if (auto v = env ("TIMEOUT_MS"); v.isNotEmpty()) j["timeoutMs"] = (std::int64_t) v.getLargeIntValue(); - if (auto v = env ("REPEAT"); v.isNotEmpty()) j["numRepeats"] = v.getIntValue(); - setFlag ("VERBOSE", "verbose"); - setFlag ("RANDOMISE", "randomiseTestOrder"); - setFlag ("SKIP_GUI_TESTS", "skipGuiTests"); - setString ("DATA_FILE", "dataFile"); - setString ("OUTPUT_DIR", "outputDir"); - setString ("OUTPUT_FILENAME", "outputFilename"); - if (auto v = env ("SAMPLE_RATES"); v.isNotEmpty()) j["sampleRates"] = settings_serializer::commaToDoubleArray (v); - if (auto v = env ("BLOCK_SIZES"); v.isNotEmpty()) j["blockSizes"] = settings_serializer::commaToIntArray (v); - setString ("RTCHECK", "realtimeCheck"); - - return j; - } + //============================================================================== + juce::String getVersionString() + { + return juce::String ("pluginval") + " - " + VERSION; } //============================================================================== @@ -303,38 +168,97 @@ Precedence is command-line options > environment variables > --config file. return raw; } - PluginvalSettings resolveSettings (const juce::StringArray& tokens, const EnvProvider& env) + //============================================================================== + ParseResult parseTokens (const juce::StringArray& tokens) { + ParseResult result; + auto& s = result.settings; + // A base64 JSON handoff from the parent process is fully authoritative. if (const auto b64 = valueForOption (tokens, "--config-base64"); b64.isNotEmpty()) { - auto s = settings_serializer::fromJsonString (decodeBase64 (b64).toStdString()); + s = settings_serializer::fromJsonString (decodeBase64 (b64).toStdString()); s.validatePath = resolvePluginPath (juce::String (s.validatePath)).toStdString(); - return s; + return result; } - auto configJson = nlohmann::json::object(); - + // Seed from --config first; CLI11 only overwrites members whose flag/env + // was provided, giving precedence: defaults < config < env < CLI. if (const auto configPath = valueForOption (tokens, "--config"); configPath.isNotEmpty()) - configJson = nlohmann::json::parse (juce::File (configPath).loadFileAsString().toStdString()); + s = settings_serializer::fromJsonFile (juce::File (configPath)); + + CLI::App app { "Validate plugins to test compatibility with hosts and verify plugin API conformance" }; + app.set_version_flag ("--version", getVersionString().toStdString()); + app.footer (getFooterText().toStdString()); + + std::string configSink; // accepted here; the file is loaded above + app.add_option ("--config", configSink, "Path to a JSON settings file (overridden by env vars and CLI options)."); + + app.add_option ("--validate", s.validatePath, "Validates the plugin at the given path (or AU id)."); + app.add_option ("--strictness-level", s.strictnessLevel, "Strictness level 1-10 (default 5).")->envname ("STRICTNESS_LEVEL"); + app.add_option ("--timeout-ms", s.timeoutMs, "Test timeout in ms (default 30000, -1 to never timeout).")->envname ("TIMEOUT_MS"); + app.add_option ("--repeat", s.numRepeats, "Number of times to repeat the tests.")->envname ("REPEAT"); + app.add_flag ("--randomise", s.randomiseTestOrder, "Run the tests in a random order per repeat.")->envname ("RANDOMISE"); + app.add_flag ("--verbose", s.verbose, "Output additional logging information.")->envname ("VERBOSE"); + app.add_flag ("--skip-gui-tests", s.skipGuiTests, "Avoid tests that create GUI windows (for headless CI).")->envname ("SKIP_GUI_TESTS"); + app.add_option ("--sample-rates", s.sampleRates, "Comma-separated sample rates (default 44100,48000,96000).")->delimiter (',')->envname ("SAMPLE_RATES"); + app.add_option ("--block-sizes", s.blockSizes, "Comma-separated block sizes (default 64,128,256,512,1024).")->delimiter (',')->envname ("BLOCK_SIZES"); + app.add_option ("--data-file", s.dataFile, "Path to a data file tests can use to configure themselves.")->envname ("DATA_FILE"); + app.add_option ("--output-dir", s.outputDir, "Directory in which to write the log files.")->envname ("OUTPUT_DIR"); + app.add_option ("--output-filename", s.outputFilename, "Filename to write logs into.")->envname ("OUTPUT_FILENAME"); + + app.add_option_function ("--disabled-tests", + [&s] (const std::string& v) { s.disabledTests = settings_serializer::disabledTestsToList (juce::String (v)); }, + "Comma-separated test names, or a path to a file listing them."); + + app.add_option_function ("--random-seed", + [&s] (const std::string& v) + { + try { s.randomSeed = settings_serializer::parseRandomSeed (juce::String (v)); } + catch (const std::exception& e) { throw CLI::ValidationError ("--random-seed", e.what()); } + }, + "Random seed (hex 0x.. or int) for replicable test runs.")->envname ("RANDOM_SEED"); + + app.add_option ("--rtcheck", s.realtimeCheck, "Real-time safety checks: disabled, enabled or relaxed.") + ->transform (CLI::CheckedTransformer (std::map { + { "disabled", RealtimeCheck::disabled }, + { "enabled", RealtimeCheck::enabled }, + { "relaxed", RealtimeCheck::relaxed } }, CLI::ignore_case)) + ->envname ("RTCHECK"); + + // Build argv (CLI11 treats element 0 as the program name) + std::vector storage; + storage.reserve ((size_t) tokens.size() + 1); + storage.emplace_back ("pluginval"); - const auto envJson = envLayer (env); - const auto cliJson = parseCliLayer (tokens); + for (const auto& t : tokens) + storage.push_back (t.toStdString()); - // Precedence (later wins): defaults < config < env < CLI. - auto merged = nlohmann::json::object(); - merged.merge_patch (configJson); - merged.merge_patch (envJson); - merged.merge_patch (cliJson); + std::vector argv; + argv.reserve (storage.size()); + + for (const auto& str : storage) + argv.push_back (str.c_str()); + + try + { + app.parse ((int) argv.size(), argv.data()); + } + catch (const CLI::ParseError& e) + { + // Includes CallForHelp / CallForVersion (exit code 0) and real errors. + result.exitCode = app.exit (e); + result.handled = true; + return result; + } - auto s = merged.get(); s.validatePath = resolvePluginPath (juce::String (s.validatePath)).toStdString(); - return s; + return result; } - PluginvalSettings parse (const juce::String& commandLine, const EnvProvider& env) + PluginvalSettings parse (const juce::String& commandLine) { - return resolveSettings (preprocess (commandLine), env); + return parseTokens (preprocess (commandLine)).settings; } //============================================================================== @@ -348,16 +272,4 @@ Precedence is command-line options > environment variables > --config file. args.addArray ({ "--config-base64", b64, "--validate", fileOrID }); return args; } - - //============================================================================== - void printHelp (const juce::String& exeName) - { - std::vector storage { exeName.toStdString(), "--help" }; - std::vector views (storage.begin(), storage.end()); - - // magic_args prints the auto-generated usage to stdout and returns HelpRequested. - (void) magic_args::parse (std::span (views), makeProgramInfo()); - - std::cout << "\n" << getTrailerText() << std::endl; - } } diff --git a/Source/SettingsParser.h b/Source/SettingsParser.h index 302457b..09cabe4 100644 --- a/Source/SettingsParser.h +++ b/Source/SettingsParser.h @@ -17,21 +17,17 @@ #include "PluginvalSettings.h" #include "PluginTests.h" -#include - /** The command-line -> settings pipeline. - Raw command line is preprocessed into argv tokens, then each input layer - (config file, environment, CLI) becomes a sparse JSON object. The layers are - merged (CLI wins last) and deserialised into a single PluginvalSettings. + CLI11 binds each option directly to a member of a single PluginvalSettings + instance (environment variables via ->envname() on the same line). A + --config JSON file seeds the struct before parsing so precedence is + defaults < config < env < CLI. Conversion to the JUCE-flavoured + PluginTests::Options happens at the boundary via toPluginTestOptions(). */ namespace settings_parser { - /** Returns the value of an environment variable, or "" if unset. */ - using EnvProvider = std::function; - juce::String systemEnv (const juce::String& name); - //============================================================================== /** Tokenises + preprocesses a raw command line: rewrites the deprecated "strictnessLevel", strips the macOS "-NSDocumentRevisionsDebugMode YES" @@ -43,30 +39,32 @@ namespace settings_parser /** True if the tokens contain a recognised command that triggers CLI mode. */ bool isCommandLine (const juce::StringArray& tokens); - //============================================================================== - /** Resolves the merged settings from preprocessed tokens + environment. */ - PluginvalSettings resolveSettings (const juce::StringArray& tokens, const EnvProvider& env = systemEnv); - - /** Convenience: preprocess + resolveSettings from a raw command line. */ - PluginvalSettings parse (const juce::String& commandLine, const EnvProvider& env = systemEnv); - /** Resolves a relative/home plugin path against the working directory, leaving absolute paths and bare component IDs untouched. */ juce::String resolvePluginPath (const juce::String& raw); + //============================================================================== + /** The outcome of parsing the option tokens. */ + struct ParseResult + { + PluginvalSettings settings; + bool handled = false; /**< true if --help/--version/parse-error was handled and the app should exit. */ + int exitCode = 0; /**< the exit code to use when handled is true. */ + }; + + /** Parses preprocessed option tokens into settings (and handles --help/--version). */ + ParseResult parseTokens (const juce::StringArray& tokens); + + /** Convenience: preprocess + parse from a raw command line, returning settings. */ + PluginvalSettings parse (const juce::String& commandLine); + //============================================================================== /** Serialises options for the child validation process as a base64 JSON handoff: { exe, --config-base64 , --validate }. */ juce::StringArray createChildProcessCommandLine (const juce::String& fileOrID, const PluginTests::Options&); - //============================================================================== - /** Prints the auto-generated usage (magic_args) plus the environment-variable - and commands trailer to stdout. - */ - void printHelp (const juce::String& exeName); - /** The "--version" output string. */ juce::String getVersionString(); } diff --git a/Source/SettingsSerializer.cpp b/Source/SettingsSerializer.cpp index 9eb62c4..a390b4d 100644 --- a/Source/SettingsSerializer.cpp +++ b/Source/SettingsSerializer.cpp @@ -45,29 +45,9 @@ namespace settings_serializer return raw.getLargeIntValue(); } - nlohmann::json commaToDoubleArray (const juce::String& raw) + std::vector disabledTestsToList (const juce::String& raw) { - auto out = nlohmann::json::array(); - - for (const auto& token : juce::StringArray::fromTokens (raw, ",", "\"")) - out.push_back (token.getDoubleValue()); - - return out; - } - - nlohmann::json commaToIntArray (const juce::String& raw) - { - auto out = nlohmann::json::array(); - - for (const auto& token : juce::StringArray::fromTokens (raw, ",", "\"")) - out.push_back (token.getIntValue()); - - return out; - } - - nlohmann::json disabledTestsToArray (const juce::String& raw) - { - auto out = nlohmann::json::array(); + std::vector out; if (juce::File::isAbsolutePath (raw)) { diff --git a/Source/SettingsSerializer.h b/Source/SettingsSerializer.h index 845d298..a5c9336 100644 --- a/Source/SettingsSerializer.h +++ b/Source/SettingsSerializer.h @@ -19,11 +19,11 @@ #include #include +#include /** - JSON (de)serialisation for PluginvalSettings plus the small value coercions - shared by the CLI and environment-variable layers (which arrive as raw - strings and must become typed JSON values). + JSON (de)serialisation for PluginvalSettings plus the couple of value + conversions the CLI parser still needs (hex seed, disabled-tests file). */ namespace settings_serializer { @@ -43,14 +43,8 @@ namespace settings_serializer */ std::int64_t parseRandomSeed (const juce::String& raw); - /** Splits a comma-separated list into a JSON array of doubles. */ - nlohmann::json commaToDoubleArray (const juce::String& raw); - - /** Splits a comma-separated list into a JSON array of ints. */ - nlohmann::json commaToIntArray (const juce::String& raw); - /** Resolves --disabled-tests: an absolute path is read line-by-line, otherwise - the value is treated as a comma-separated list. Returns a JSON string array. + the value is treated as a comma-separated list. */ - nlohmann::json disabledTestsToArray (const juce::String& raw); + std::vector disabledTestsToList (const juce::String& raw); } diff --git a/docs/Command line options.md b/docs/Command line options.md index 3af2dc2..1267c53 100644 --- a/docs/Command line options.md +++ b/docs/Command line options.md @@ -1,48 +1,56 @@ -Usage: pluginval [OPTIONS...] -Validate plugins to test compatibility with hosts and verify plugin API conformance - -Examples: - - pluginval --strictness-level 5 --validate path/to/plugin - pluginval path/to/plugin - pluginval --config settings.json --validate path/to/plugin - -Options: - - --validate=VALUE Validates the plugin at the given path (or AU id). Optional if the path is the last argument. - --strictness-level=VALUE - Strictness level 1-10 (default 5, minimum 5 recommended). - --random-seed=VALUE Random seed (hex 0x.. or int) for replicable test runs. - --timeout-ms=VALUE Test timeout in ms (default 30000, -1 to never timeout). - --repeat=VALUE Number of times to repeat the tests. - --randomise Run the tests in a random order per repeat. - --verbose Output additional logging information. - --skip-gui-tests Avoid tests that create GUI windows (for headless CI). - --sample-rates=VALUE Comma-separated sample rates (default 44100,48000,96000). - --block-sizes=VALUE Comma-separated block sizes (default 64,128,256,512,1024). - --data-file=VALUE Path to a data file tests can use to configure themselves. - --output-dir=VALUE Directory in which to write the log files. - --output-filename=VALUE Filename to write logs into. - --disabled-tests=VALUE Comma-separated test names, or a path to a file listing them. - --rtcheck=VALUE Real-time safety checks: disabled, enabled or relaxed. - --config=VALUE Path to a JSON file of settings (overridden by env vars and CLI options). - - -?, --help show this message - --version print program version +Validate plugins to test compatibility with hosts and verify plugin API +conformance + + +pluginval [OPTIONS] + + +OPTIONS: + -h, --help Print this help message and exit + --version Display program version information and exit + --config TEXT Path to a JSON settings file (overridden by env vars and CLI + options). + --validate TEXT Validates the plugin at the given path (or AU id). + --strictness-level INT (Env:STRICTNESS_LEVEL) + Strictness level 1-10 (default 5). + --timeout-ms INT (Env:TIMEOUT_MS) + Test timeout in ms (default 30000, -1 to never timeout). + --repeat INT (Env:REPEAT) + Number of times to repeat the tests. + --randomise (Env:RANDOMISE) + Run the tests in a random order per repeat. + --verbose (Env:VERBOSE) + Output additional logging information. + --skip-gui-tests (Env:SKIP_GUI_TESTS) + Avoid tests that create GUI windows (for headless CI). + --sample-rates FLOAT ... (Env:SAMPLE_RATES) + Comma-separated sample rates (default 44100,48000,96000). + --block-sizes INT ... (Env:BLOCK_SIZES) + Comma-separated block sizes (default 64,128,256,512,1024). + --data-file TEXT (Env:DATA_FILE) + Path to a data file tests can use to configure themselves. + --output-dir TEXT (Env:OUTPUT_DIR) + Directory in which to write the log files. + --output-filename TEXT (Env:OUTPUT_FILENAME) + Filename to write logs into. + --disabled-tests TEXT + Comma-separated test names, or a path to a file listing them. + --random-seed TEXT (Env:RANDOM_SEED) + Random seed (hex 0x.. or int) for replicable test runs. + --rtcheck ENUM:value in {disabled->0,enabled->1,relaxed->2} OR {0,1,2} (Env:RTCHECK) + Real-time safety checks: disabled, enabled or relaxed. Other commands: - --version Print the pluginval version. - --run-tests Run the internal unit tests. - --strictness-help [level] List all tests that run at the given strictness level. +--run-tests Run the internal unit tests. +--strictness-help [level] List all tests that run at the given strictness level. Exit code: - 0 if all tests complete successfully - 1 if there are any errors - -Additionally, you can specify any of the command line options as environment -variables by removing prefix dashes, converting internal dashes to underscores -and capitalising all letters, e.g. - "--skip-gui-tests" > "SKIP_GUI_TESTS=1" - "--timeout-ms 30000" > "TIMEOUT_MS=30000" -Precedence is command-line options > environment variables > --config file. - +0 if all tests complete successfully +1 if there are any errors + +You can also specify any option as an environment variable by removing the +prefix +dashes, converting internal dashes to underscores and capitalising, e.g. +"--skip-gui-tests" -> "SKIP_GUI_TESTS=1" +"--timeout-ms 30000" -> "TIMEOUT_MS=30000" +Precedence: command-line options > environment variables > --config file. From 6f7344a8997471344891b1e213cbecaee57f6bcb Mon Sep 17 00:00:00 2001 From: David Rowland Date: Sun, 7 Jun 2026 09:44:16 +0100 Subject: [PATCH 05/12] Fix CI failures from the CLI11 refactor Three regressions surfaced by CI (develop is green): - Linux build: std::int64_t (== long on LP64) vs juce::int64 (== long long) made expectEquals template deduction fail. Cast the settings int64 fields to juce::int64 in CommandLineTests. (macOS compiled because there the two types are identical.) - "Pluginval as Dependency" Verify: verify_pluginval runs `pluginval --help | grep "JUCE v"`. CLI11's auto help didn't print the JUCE version; add it to the help footer. - Windows --run-tests exit 127: runUnitTests() called the [[noreturn]] juce::ConsoleApplication::fail(), which throws; outside a ConsoleApplication command handler that throw was uncaught -> std::terminate. Set the application return value directly instead. Co-Authored-By: Claude Opus 4.8 --- Source/CommandLine.cpp | 8 +++++++- Source/CommandLineTests.cpp | 14 +++++++------- Source/SettingsParser.cpp | 2 +- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/Source/CommandLine.cpp b/Source/CommandLine.cpp index e561d39..a3065fb 100644 --- a/Source/CommandLine.cpp +++ b/Source/CommandLine.cpp @@ -171,8 +171,14 @@ static void runUnitTests() testRunner.runTestsInCategory ("pluginval"); const int numFailures = getNumTestFailures (testRunner); + // Set the return value directly rather than juce::ConsoleApplication::fail(), + // which throws and would terminate the process when called outside a + // ConsoleApplication command handler. if (numFailures > 0) - juce::ConsoleApplication::fail (juce::String (numFailures) + " tests failed!!!"); + { + std::cout << numFailures << " tests failed!!!" << std::endl; + juce::JUCEApplication::getInstance()->setApplicationReturnValue (1); + } } //============================================================================== diff --git a/Source/CommandLineTests.cpp b/Source/CommandLineTests.cpp index 9d22345..59deecf 100644 --- a/Source/CommandLineTests.cpp +++ b/Source/CommandLineTests.cpp @@ -94,14 +94,14 @@ struct CommandLineTests : public juce::UnitTest beginTest ("Negative timeout"); { - expectEquals (parse ("--timeout-ms -1 --validate x").timeoutMs, (juce::int64) -1); + expectEquals ((juce::int64) parse ("--timeout-ms -1 --validate x").timeoutMs, (juce::int64) -1); } beginTest ("Command line random (hex and int)"); { - expectEquals (parse ("--random-seed 0x7f2da1 --validate x").randomSeed, (juce::int64) 8334753); - expectEquals (parse ("--random-seed 0x692bc1f --validate x").randomSeed, (juce::int64) 110279711); - expectEquals (parse ("--random-seed 1234 --validate x").randomSeed, (juce::int64) 1234); + expectEquals ((juce::int64) parse ("--random-seed 0x7f2da1 --validate x").randomSeed, (juce::int64) 8334753); + expectEquals ((juce::int64) parse ("--random-seed 0x692bc1f --validate x").randomSeed, (juce::int64) 110279711); + expectEquals ((juce::int64) parse ("--random-seed 1234 --validate x").randomSeed, (juce::int64) 1234); } beginTest ("Comma-separated lists"); @@ -251,7 +251,7 @@ struct CommandLineTests : public juce::UnitTest { const auto s = parse (cfg + " --validate x"); expectEquals (s.strictnessLevel, 2); - expectEquals (s.timeoutMs, (juce::int64) 12345); + expectEquals ((juce::int64) s.timeoutMs, (juce::int64) 12345); expectEquals (s.numRepeats, 4); } @@ -261,7 +261,7 @@ struct CommandLineTests : public juce::UnitTest const auto s = parse (cfg + " --validate x"); clearKnownEnv(); expectEquals (s.strictnessLevel, 6); // env beats config - expectEquals (s.timeoutMs, (juce::int64) 12345); // still from config + expectEquals ((juce::int64) s.timeoutMs, (juce::int64) 12345); // still from config } // CLI overrides env and config @@ -270,7 +270,7 @@ struct CommandLineTests : public juce::UnitTest const auto s = parse (cfg + " --strictness-level 9 --validate x"); clearKnownEnv(); expectEquals (s.strictnessLevel, 9); - expectEquals (s.timeoutMs, (juce::int64) 12345); + expectEquals ((juce::int64) s.timeoutMs, (juce::int64) 12345); } } diff --git a/Source/SettingsParser.cpp b/Source/SettingsParser.cpp index 3c2bcb3..01bf726 100644 --- a/Source/SettingsParser.cpp +++ b/Source/SettingsParser.cpp @@ -88,7 +88,7 @@ namespace settings_parser juce::String getFooterText() { - return juce::String ( + return juce::SystemStats::getJUCEVersion() + "\n\n" + juce::String ( R"(Other commands: --run-tests Run the internal unit tests. --strictness-help [level] List all tests that run at the given strictness level. From c1951aa1cfff389feb01c09080fd704bb9fbe093 Mon Sep 17 00:00:00 2001 From: David Rowland Date: Sun, 7 Jun 2026 09:57:48 +0100 Subject: [PATCH 06/12] Fix Linux X11 macro clash; print unit-test failures to stdout - Linux build: include before the JUCE headers. JUCE pulls in X11 on Linux (JUCE_GUI_BASICS_INCLUDE_XHEADERS), whose `#define Success 0` clashed with CLI11's CLI::ExitCodes::Success enumerator. macOS/Windows don't use X11, so only Linux was affected. - runUnitTests now prints each failed test (name + messages) to stdout. juce::UnitTestRunner logs via juce::Logger, which on a GUI app (Windows) doesn't reach the console, so CI couldn't show which tests failed. Co-Authored-By: Claude Opus 4.8 --- Source/CommandLine.cpp | 26 +++++++++++++++++--------- Source/SettingsParser.cpp | 7 +++++-- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/Source/CommandLine.cpp b/Source/CommandLine.cpp index a3065fb..f152d27 100644 --- a/Source/CommandLine.cpp +++ b/Source/CommandLine.cpp @@ -154,22 +154,30 @@ static void printStrictnessHelp (int level) std::cout << std::endl; } -static int getNumTestFailures (juce::UnitTestRunner& testRunner) +static void runUnitTests() { + juce::UnitTestRunner testRunner; + testRunner.runTestsInCategory ("pluginval"); + int numFailures = 0; + // Print failures to stdout: juce::UnitTestRunner logs via juce::Logger, which + // on a GUI app (e.g. Windows) doesn't reach the console. for (int i = 0; i < testRunner.getNumResults(); ++i) - if (auto result = testRunner.getResult (i)) + { + if (auto* result = testRunner.getResult (i)) + { numFailures += result->failures; - return numFailures; -} + if (result->failures > 0) + { + std::cout << "!!! FAILED: " << result->unitTestName << " / " << result->subcategoryName << std::endl; -static void runUnitTests() -{ - juce::UnitTestRunner testRunner; - testRunner.runTestsInCategory ("pluginval"); - const int numFailures = getNumTestFailures (testRunner); + for (const auto& message : result->messages) + std::cout << " " << message << std::endl; + } + } + } // Set the return value directly rather than juce::ConsoleApplication::fail(), // which throws and would terminate the process when called outside a diff --git a/Source/SettingsParser.cpp b/Source/SettingsParser.cpp index 01bf726..6af0f8d 100644 --- a/Source/SettingsParser.cpp +++ b/Source/SettingsParser.cpp @@ -12,11 +12,14 @@ ==============================================================================*/ +// CLI11 must be included before the JUCE headers: on Linux JUCE pulls in the X11 +// headers (JUCE_GUI_BASICS_INCLUDE_XHEADERS), which #define Success/None/Bool/etc. +// and would clash with CLI11's CLI::ExitCodes::Success enumerator. +#include + #include "SettingsParser.h" #include "SettingsSerializer.h" -#include - #include #include #include From 54dd719bc09ad851c9f037fea508e389ba0282b7 Mon Sep 17 00:00:00 2001 From: David Rowland Date: Sun, 7 Jun 2026 10:21:05 +0100 Subject: [PATCH 07/12] Fix Windows test: compare raw parsed paths, not juce::File-normalised "Command line parser" asserted opts.dataFile/outputDir.getFullPathName() against Unix-style literals; on Windows juce::File normalises those to the current drive (e.g. "D:\path\to\file"), failing the comparison. Assert the raw parsed settings strings instead, which are identical on all platforms. Co-Authored-By: Claude Opus 4.8 --- Source/CommandLineTests.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Source/CommandLineTests.cpp b/Source/CommandLineTests.cpp index 59deecf..2270b2d 100644 --- a/Source/CommandLineTests.cpp +++ b/Source/CommandLineTests.cpp @@ -87,8 +87,10 @@ struct CommandLineTests : public juce::UnitTest expectEquals (opts.randomSeed, (juce::int64) 1234); expectEquals (opts.timeoutMs, (juce::int64) 20000); expectEquals (opts.numRepeats, 11); - expectEquals (opts.dataFile.getFullPathName(), juce::String ("/path/to/file")); - expectEquals (opts.outputDir.getFullPathName(), juce::String ("/path/to/dir")); + // Compare the raw parsed strings: juce::File would normalise these to the + // current drive on Windows (e.g. "D:\path\to\file"). + expectEquals (juce::String (settings.dataFile), juce::String ("/path/to/file")); + expectEquals (juce::String (settings.outputDir), juce::String ("/path/to/dir")); expectEquals (juce::String (settings.validatePath), juce::String ("/path/to/plugin")); } From a3c906961577cc9934f931a76c6619a277b7ad0f Mon Sep 17 00:00:00 2001 From: David Rowland Date: Sun, 7 Jun 2026 13:08:45 +0100 Subject: [PATCH 08/12] Revise precedence to defaults < env < config < CLI Reworks the layering so a --config file (deliberate, per-invocation) ranks above ambient environment variables, and makes --config repeatable. - Environment layer: env-var names are now derived from the registered CLI11 options (--strictness-level -> STRICTNESS_LEVEL) instead of CLI11's ->envname(), so there is no hand-maintained env table and env support is automatic for every option. A synthetic "--name=value" argv is built from the environment and parsed by CLI11, reusing all its coercion (comma lists, enum transformer, hex-seed callback). Flags accept --flag=1/0. - --config: repeatable; each JSON file is merge_patch-ed in command-line order (later files win per key). Applied between the env and CLI layers, so it beats env and loses to explicit CLI options. (Inline JSON dropped for now to avoid command-line re-tokenisation hazards; file paths only.) - The env layer reads through an injectable EnvProvider again, so the unit tests no longer mutate the real process environment (also removes the Windows _putenv_s fragility). - Tests updated for the new precedence + a repeatable-config test. - Help footer, docs and CLAUDE.md updated. Verified end-to-end: env(6) -> 6; env(6)+config(2) -> 2; +CLI(9) -> 9. Co-Authored-By: Claude Opus 4.8 --- CLAUDE.md | 32 +++-- Source/CommandLineTests.cpp | 126 +++++++++--------- Source/SettingsParser.cpp | 252 +++++++++++++++++++++++++---------- Source/SettingsParser.h | 21 ++- docs/Command line options.md | 44 +++--- 5 files changed, 300 insertions(+), 175 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 3e00941..f466f3f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -194,24 +194,32 @@ VST2_SDK_DIR=/path/to/vst2sdk cmake -B Builds/Debug . ### CLI Settings Pipeline Command-line parsing centres on one plain settings struct (`PluginvalSettings`) -that CLI11 binds to directly. The flow (in `SettingsParser`): +that CLI11 binds to directly. A single instance is filled by successive layers, +**lowest to highest precedence: defaults → environment → `--config` → CLI** +(in `SettingsParser::parseTokens`): 1. **preprocess** the raw command line — rewrite the deprecated `strictnessLevel`, strip the macOS `-NSDocumentRevisionsDebugMode YES` flag, and insert an implicit `--validate` when the last argument is a bare plugin path. -2. If `--config ` is present, **seed** the struct from that JSON first. -3. **CLI11** binds each `add_option`/`add_flag` straight to a `PluginvalSettings` - member, with the environment variable on the same line via `->envname()`. - Because CLI11 only overwrites a member when its flag/env was actually provided, - precedence is **defaults < config < env < CLI** (CLI wins) with no manual - layering. Comma lists use `->delimiter(',')`, the enum uses a - `CheckedTransformer`, and the hex/int seed is a small callback. -4. `PluginvalSettings::toPluginTestOptions()` converts to the JUCE-flavoured - `PluginTests::Options` at the boundary. +2. **Environment layer.** Env-var names are *derived* from the registered + options (`--strictness-level` → `STRICTNESS_LEVEL`), so there is no separate + env table. A synthetic `--name=value` argv is built from the environment and + parsed by CLI11, reusing all its coercion. +3. **`--config` layer.** Repeatable; each JSON file is `merge_patch`-ed in + command-line order (later files win per key). Beats the environment. +4. **CLI layer.** The real arguments are parsed last and beat everything; + CLI11 only overwrites a member when its option was actually provided. + +`configureApp()` registers every option (bound to the struct) and is used for +both the env pass and the CLI pass. Comma lists use `->delimiter(',')`, the enum +uses a `CheckedTransformer`, and the hex/int seed is a small callback. +`PluginvalSettings::toPluginTestOptions()` converts to the JUCE-flavoured +`PluginTests::Options` at the boundary. Adding a new option is three edits: a struct member, an entry in the nlohmann -macro list, and one `add_option(...)` line. `SettingsSerializer` handles JSON -load/save plus the two remaining conversions (hex seed, disabled-tests file). +macro list, and one `add_option(...)` line — its environment variable then works +automatically. `SettingsSerializer` handles JSON load/save plus the two +remaining conversions (hex seed, disabled-tests file). The child validation process receives a fully-resolved, **authoritative** settings set via a base64-encoded JSON argument (`--config-base64`), avoiding diff --git a/Source/CommandLineTests.cpp b/Source/CommandLineTests.cpp index 2270b2d..24f3d14 100644 --- a/Source/CommandLineTests.cpp +++ b/Source/CommandLineTests.cpp @@ -12,7 +12,7 @@ ==============================================================================*/ -#include +#include struct CommandLineTests : public juce::UnitTest { @@ -21,46 +21,32 @@ struct CommandLineTests : public juce::UnitTest { } - static constexpr const char* knownEnvVars[] = { - "STRICTNESS_LEVEL", "RANDOM_SEED", "TIMEOUT_MS", "VERBOSE", "REPEAT", - "RANDOMISE", "SKIP_GUI_TESTS", "DATA_FILE", "OUTPUT_DIR", "OUTPUT_FILENAME", - "SAMPLE_RATES", "BLOCK_SIZES", "RTCHECK" - }; - - static void setEnv (const char* name, const char* value) + static settings_parser::EnvProvider emptyEnv() { - #if JUCE_WINDOWS - _putenv_s (name, value); - #else - ::setenv (name, value, 1); - #endif + return [] (const juce::String&) { return juce::String(); }; } - static void unsetEnv (const char* name) + static settings_parser::EnvProvider envFrom (std::map vars) { - #if JUCE_WINDOWS - _putenv_s (name, ""); - #else - ::unsetenv (name); - #endif + return [vars = std::move (vars)] (const juce::String& name) + { + const auto it = vars.find (name); + return it != vars.end() ? it->second : juce::String(); + }; } - static void clearKnownEnv() + static PluginvalSettings parse (const juce::String& cmd, settings_parser::EnvProvider env) { - for (auto* n : knownEnvVars) - unsetEnv (n); + return settings_parser::parse (cmd, env); } static PluginvalSettings parse (const juce::String& cmd) { - return settings_parser::parse (cmd); + return settings_parser::parse (cmd, emptyEnv()); } void runTest() override { - // Start from a clean environment so host env vars don't affect the deterministic tests. - clearKnownEnv(); - beginTest ("Command line defaults"); { const auto opts = parse ("").toPluginTestOptions(); @@ -203,23 +189,23 @@ struct CommandLineTests : public juce::UnitTest beginTest ("Environment variables"); { - setEnv ("STRICTNESS_LEVEL", "7"); - setEnv ("RANDOM_SEED", "1234"); - setEnv ("TIMEOUT_MS", "20000"); - setEnv ("VERBOSE", "1"); - setEnv ("REPEAT", "11"); - setEnv ("RANDOMISE", "1"); - setEnv ("SKIP_GUI_TESTS", "1"); - setEnv ("DATA_FILE", "/path/to/file"); - setEnv ("OUTPUT_DIR", "/path/to/dir"); - setEnv ("SAMPLE_RATES", "22050,44100"); - setEnv ("BLOCK_SIZES", "32,64"); - setEnv ("RTCHECK", "relaxed"); - - const auto opts = parse ("--validate x").toPluginTestOptions(); - - clearKnownEnv(); - + const auto env = envFrom ({ + { "STRICTNESS_LEVEL", "7" }, + { "RANDOM_SEED", "1234" }, + { "TIMEOUT_MS", "20000" }, + { "VERBOSE", "1" }, + { "REPEAT", "11" }, + { "RANDOMISE", "1" }, + { "SKIP_GUI_TESTS", "1" }, + { "DATA_FILE", "/path/to/file" }, + { "OUTPUT_DIR", "/path/to/dir" }, + { "SAMPLE_RATES", "22050,44100" }, + { "BLOCK_SIZES", "32,64" }, + { "RTCHECK", "relaxed" }, + }); + + const auto settings = parse ("--validate x", env); + const auto opts = settings.toPluginTestOptions(); expectEquals (opts.strictnessLevel, 7); expectEquals (opts.randomSeed, (juce::int64) 1234); expectEquals (opts.timeoutMs, (juce::int64) 20000); @@ -227,6 +213,7 @@ struct CommandLineTests : public juce::UnitTest expectEquals (opts.numRepeats, 11); expect (opts.randomiseTestOrder); expect (opts.withGUI == false); + expectEquals (juce::String (settings.dataFile), juce::String ("/path/to/file")); expect (opts.sampleRates == std::vector ({ 22050.0, 44100.0 })); expect (opts.blockSizes == std::vector ({ 32, 64 })); expect (opts.realtimeCheck == RealtimeCheck::relaxed); @@ -234,48 +221,61 @@ struct CommandLineTests : public juce::UnitTest beginTest ("Command line overrides environment variables"); { - setEnv ("STRICTNESS_LEVEL", "3"); - const auto envOnly = parse ("--validate x").strictnessLevel; - const auto cliWins = parse ("--strictness-level 9 --validate x").strictnessLevel; - clearKnownEnv(); - - expectEquals (envOnly, 3); // env only - expectEquals (cliWins, 9); // CLI wins + const auto env = envFrom ({ { "STRICTNESS_LEVEL", "3" } }); + expectEquals (parse ("--validate x", env).strictnessLevel, 3); // env only + expectEquals (parse ("--strictness-level 9 --validate x", env).strictnessLevel, 9); // CLI wins } - beginTest ("Config file and precedence (CLI > env > config > defaults)"); + beginTest ("Precedence: CLI > --config > env > defaults"); { juce::TemporaryFile configFile (".json"); - configFile.getFile().replaceWithText (R"({ "strictnessLevel": 2, "timeoutMs": 12345, "numRepeats": 4 })"); + configFile.getFile().replaceWithText (R"({ "strictnessLevel": 2, "timeoutMs": 12345 })"); const auto cfg = "--config " + configFile.getFile().getFullPathName().quoted(); + const auto env6 = envFrom ({ { "STRICTNESS_LEVEL", "6" } }); + + // env beats defaults + expectEquals (parse ("--validate x", env6).strictnessLevel, 6); + // config alone { const auto s = parse (cfg + " --validate x"); expectEquals (s.strictnessLevel, 2); expectEquals ((juce::int64) s.timeoutMs, (juce::int64) 12345); - expectEquals (s.numRepeats, 4); } - // env overrides config + // config beats env { - setEnv ("STRICTNESS_LEVEL", "6"); - const auto s = parse (cfg + " --validate x"); - clearKnownEnv(); - expectEquals (s.strictnessLevel, 6); // env beats config - expectEquals ((juce::int64) s.timeoutMs, (juce::int64) 12345); // still from config + const auto s = parse (cfg + " --validate x", env6); + expectEquals (s.strictnessLevel, 2); // config wins over env + expectEquals ((juce::int64) s.timeoutMs, (juce::int64) 12345); } - // CLI overrides env and config + // CLI beats config and env { - setEnv ("STRICTNESS_LEVEL", "6"); - const auto s = parse (cfg + " --strictness-level 9 --validate x"); - clearKnownEnv(); + const auto s = parse (cfg + " --strictness-level 9 --validate x", env6); expectEquals (s.strictnessLevel, 9); expectEquals ((juce::int64) s.timeoutMs, (juce::int64) 12345); } } + beginTest ("Repeatable --config merges per key, last wins"); + { + juce::TemporaryFile baseFile (".json"); + baseFile.getFile().replaceWithText (R"({ "strictnessLevel": 2, "timeoutMs": 11111 })"); + + juce::TemporaryFile overrideFile (".json"); + overrideFile.getFile().replaceWithText (R"({ "strictnessLevel": 8 })"); + + const auto cmd = "--config " + baseFile.getFile().getFullPathName().quoted() + + " --config " + overrideFile.getFile().getFullPathName().quoted() + + " --validate x"; + + const auto s = parse (cmd); + expectEquals (s.strictnessLevel, 8); // overridden by the second file + expectEquals ((juce::int64) s.timeoutMs, (juce::int64) 11111); // untouched, kept from the first + } + beginTest ("Child-process command line round-trip"); { PluginTests::Options opts; diff --git a/Source/SettingsParser.cpp b/Source/SettingsParser.cpp index 6af0f8d..6d302cc 100644 --- a/Source/SettingsParser.cpp +++ b/Source/SettingsParser.cpp @@ -22,11 +22,23 @@ #include #include +#include +#include #include #include namespace settings_parser { + juce::String systemEnv (const juce::String& name) + { + return juce::SystemStats::getEnvironmentVariable (name, {}); + } + + juce::String getVersionString() + { + return juce::String ("pluginval") + " - " + VERSION; + } + //============================================================================== namespace { @@ -71,6 +83,28 @@ namespace settings_parser return {}; } + /** Returns every value supplied for an option, in command-line order. */ + juce::StringArray allValuesForOption (const juce::StringArray& tokens, juce::StringRef option) + { + juce::StringArray out; + const juce::String prefix = juce::String (option) + "="; + + for (int i = 0; i < tokens.size(); ++i) + { + if (tokens[i] == option) + { + if (i + 1 < tokens.size()) + out.add (tokens[i + 1]); + } + else if (tokens[i].startsWith (prefix)) + { + out.add (tokens[i].substring (prefix.length())); + } + } + + return out; + } + bool hasOption (const juce::StringArray& tokens, juce::StringRef option) { const juce::String prefix = juce::String (option) + "="; @@ -104,14 +138,119 @@ You can also specify any option as an environment variable by removing the prefi dashes, converting internal dashes to underscores and capitalising, e.g. "--skip-gui-tests" -> "SKIP_GUI_TESTS=1" "--timeout-ms 30000" -> "TIMEOUT_MS=30000" -Precedence: command-line options > environment variables > --config file.)"); +Precedence (lowest to highest): defaults, environment variables, --config, command-line options. +--config is repeatable; later files win per key.)"); } - } - //============================================================================== - juce::String getVersionString() - { - return juce::String ("pluginval") + " - " + VERSION; + std::string toEnvName (const std::string& longName) + { + return juce::String (longName).toUpperCase().replace ("-", "_").toStdString(); + } + + /** Loads one --config JSON file. */ + nlohmann::json loadConfigFile (const juce::String& path) + { + const juce::File file (path); + + if (! file.existsAsFile()) + throw std::runtime_error (("--config file not found: " + path).toStdString()); + + return nlohmann::json::parse (file.loadFileAsString().toStdString()); + } + + /** Registers every option, bound to s. No ->envname(): the environment is a + separate, lower-precedence layer (see parseTokens). */ + void configureApp (CLI::App& app, PluginvalSettings& s) + { + app.set_version_flag ("--version", getVersionString().toStdString()); + app.footer (getFooterText().toStdString()); + + // Accepted so the CLI parse doesn't error on --config; the values are + // handled manually (repeatable, inline-or-file) in parseTokens. + app.add_option_function> ("--config", + [] (const std::vector&) {}, + "Path to a JSON settings file. Repeatable; later files win per key.")->take_all(); + + app.add_option ("--validate", s.validatePath, "Validates the plugin at the given path (or AU id)."); + app.add_option ("--strictness-level", s.strictnessLevel, "Strictness level 1-10 (default 5)."); + app.add_option ("--timeout-ms", s.timeoutMs, "Test timeout in ms (default 30000, -1 to never timeout)."); + app.add_option ("--repeat", s.numRepeats, "Number of times to repeat the tests."); + app.add_flag ("--randomise", s.randomiseTestOrder, "Run the tests in a random order per repeat."); + app.add_flag ("--verbose", s.verbose, "Output additional logging information."); + app.add_flag ("--skip-gui-tests", s.skipGuiTests, "Avoid tests that create GUI windows (for headless CI)."); + app.add_option ("--sample-rates", s.sampleRates, "Comma-separated sample rates (default 44100,48000,96000).")->delimiter (','); + app.add_option ("--block-sizes", s.blockSizes, "Comma-separated block sizes (default 64,128,256,512,1024).")->delimiter (','); + app.add_option ("--data-file", s.dataFile, "Path to a data file tests can use to configure themselves."); + app.add_option ("--output-dir", s.outputDir, "Directory in which to write the log files."); + app.add_option ("--output-filename", s.outputFilename, "Filename to write logs into."); + + app.add_option_function ("--disabled-tests", + [&s] (const std::string& v) { s.disabledTests = settings_serializer::disabledTestsToList (juce::String (v)); }, + "Comma-separated test names, or a path to a file listing them."); + + app.add_option_function ("--random-seed", + [&s] (const std::string& v) + { + try { s.randomSeed = settings_serializer::parseRandomSeed (juce::String (v)); } + catch (const std::exception& e) { throw CLI::ValidationError ("--random-seed", e.what()); } + }, + "Random seed (hex 0x.. or int) for replicable test runs."); + + app.add_option ("--rtcheck", s.realtimeCheck, "Real-time safety checks: disabled, enabled or relaxed.") + ->transform (CLI::CheckedTransformer (std::map { + { "disabled", RealtimeCheck::disabled }, + { "enabled", RealtimeCheck::enabled }, + { "relaxed", RealtimeCheck::relaxed } }, CLI::ignore_case)); + } + + /** Builds a synthetic argv from the environment by deriving an env-var name + from each registered option (e.g. --strictness-level -> STRICTNESS_LEVEL). + CLI11 then parses and coerces it like any other argument. */ + std::vector buildEnvArgv (const CLI::App& app, const EnvProvider& env) + { + std::vector argv { "pluginval" }; + + for (const auto* opt : app.get_options()) + { + const auto& lnames = opt->get_lnames(); + + if (lnames.empty()) + continue; + + const auto& lname = lnames.front(); + + // Skip the meta options that shouldn't be environment-driven. + if (lname == "help" || lname == "version" || lname == "config" || lname == "validate") + continue; + + if (const auto value = env (juce::String (toEnvName (lname))); value.isNotEmpty()) + argv.push_back ("--" + lname + "=" + value.toStdString()); // works for flags too (--flag=1/0) + } + + return argv; + } + + /** Runs one parse pass. Returns an exit code if the parse was "handled" + (help/version/error), or nullopt on success. */ + std::optional runParse (CLI::App& app, const std::vector& argv) + { + std::vector cargv; + cargv.reserve (argv.size()); + + for (const auto& a : argv) + cargv.push_back (a.c_str()); + + try + { + app.parse ((int) cargv.size(), cargv.data()); + } + catch (const CLI::ParseError& e) + { + return app.exit (e); + } + + return std::nullopt; + } } //============================================================================== @@ -172,7 +311,7 @@ Precedence: command-line options > environment variables > --config file.)"); } //============================================================================== - ParseResult parseTokens (const juce::StringArray& tokens) + ParseResult parseTokens (const juce::StringArray& tokens, const EnvProvider& env) { ParseResult result; auto& s = result.settings; @@ -185,83 +324,56 @@ Precedence: command-line options > environment variables > --config file.)"); return result; } - // Seed from --config first; CLI11 only overwrites members whose flag/env - // was provided, giving precedence: defaults < config < env < CLI. - if (const auto configPath = valueForOption (tokens, "--config"); configPath.isNotEmpty()) - s = settings_serializer::fromJsonFile (juce::File (configPath)); - - CLI::App app { "Validate plugins to test compatibility with hosts and verify plugin API conformance" }; - app.set_version_flag ("--version", getVersionString().toStdString()); - app.footer (getFooterText().toStdString()); - - std::string configSink; // accepted here; the file is loaded above - app.add_option ("--config", configSink, "Path to a JSON settings file (overridden by env vars and CLI options)."); - - app.add_option ("--validate", s.validatePath, "Validates the plugin at the given path (or AU id)."); - app.add_option ("--strictness-level", s.strictnessLevel, "Strictness level 1-10 (default 5).")->envname ("STRICTNESS_LEVEL"); - app.add_option ("--timeout-ms", s.timeoutMs, "Test timeout in ms (default 30000, -1 to never timeout).")->envname ("TIMEOUT_MS"); - app.add_option ("--repeat", s.numRepeats, "Number of times to repeat the tests.")->envname ("REPEAT"); - app.add_flag ("--randomise", s.randomiseTestOrder, "Run the tests in a random order per repeat.")->envname ("RANDOMISE"); - app.add_flag ("--verbose", s.verbose, "Output additional logging information.")->envname ("VERBOSE"); - app.add_flag ("--skip-gui-tests", s.skipGuiTests, "Avoid tests that create GUI windows (for headless CI).")->envname ("SKIP_GUI_TESTS"); - app.add_option ("--sample-rates", s.sampleRates, "Comma-separated sample rates (default 44100,48000,96000).")->delimiter (',')->envname ("SAMPLE_RATES"); - app.add_option ("--block-sizes", s.blockSizes, "Comma-separated block sizes (default 64,128,256,512,1024).")->delimiter (',')->envname ("BLOCK_SIZES"); - app.add_option ("--data-file", s.dataFile, "Path to a data file tests can use to configure themselves.")->envname ("DATA_FILE"); - app.add_option ("--output-dir", s.outputDir, "Directory in which to write the log files.")->envname ("OUTPUT_DIR"); - app.add_option ("--output-filename", s.outputFilename, "Filename to write logs into.")->envname ("OUTPUT_FILENAME"); - - app.add_option_function ("--disabled-tests", - [&s] (const std::string& v) { s.disabledTests = settings_serializer::disabledTestsToList (juce::String (v)); }, - "Comma-separated test names, or a path to a file listing them."); - - app.add_option_function ("--random-seed", - [&s] (const std::string& v) + // 1. Environment layer (env-var names derived from the registered options). + { + CLI::App envApp; + configureApp (envApp, s); + + if (const auto code = runParse (envApp, buildEnvArgv (envApp, env)); code) { - try { s.randomSeed = settings_serializer::parseRandomSeed (juce::String (v)); } - catch (const std::exception& e) { throw CLI::ValidationError ("--random-seed", e.what()); } - }, - "Random seed (hex 0x.. or int) for replicable test runs.")->envname ("RANDOM_SEED"); + result.exitCode = *code; + result.handled = true; + return result; + } + } - app.add_option ("--rtcheck", s.realtimeCheck, "Real-time safety checks: disabled, enabled or relaxed.") - ->transform (CLI::CheckedTransformer (std::map { - { "disabled", RealtimeCheck::disabled }, - { "enabled", RealtimeCheck::enabled }, - { "relaxed", RealtimeCheck::relaxed } }, CLI::ignore_case)) - ->envname ("RTCHECK"); + // 2. --config layer: repeatable, inline JSON or file, merged per key in + // command-line order (last wins). Beats the environment, loses to CLI. + if (const auto configs = allValuesForOption (tokens, "--config"); ! configs.isEmpty()) + { + auto merged = settings_serializer::toJson (s); - // Build argv (CLI11 treats element 0 as the program name) - std::vector storage; - storage.reserve ((size_t) tokens.size() + 1); - storage.emplace_back ("pluginval"); + for (const auto& source : configs) + merged.merge_patch (loadConfigFile (source)); - for (const auto& t : tokens) - storage.push_back (t.toStdString()); + s = merged.get(); + } - std::vector argv; - argv.reserve (storage.size()); + // 3. Command-line layer: the individual options beat everything. + { + CLI::App cliApp { "Validate plugins to test compatibility with hosts and verify plugin API conformance" }; + configureApp (cliApp, s); - for (const auto& str : storage) - argv.push_back (str.c_str()); + std::vector argv { "pluginval" }; - try - { - app.parse ((int) argv.size(), argv.data()); - } - catch (const CLI::ParseError& e) - { - // Includes CallForHelp / CallForVersion (exit code 0) and real errors. - result.exitCode = app.exit (e); - result.handled = true; - return result; + for (const auto& t : tokens) + argv.push_back (t.toStdString()); + + if (const auto code = runParse (cliApp, argv); code) + { + result.exitCode = *code; + result.handled = true; + return result; + } } s.validatePath = resolvePluginPath (juce::String (s.validatePath)).toStdString(); return result; } - PluginvalSettings parse (const juce::String& commandLine) + PluginvalSettings parse (const juce::String& commandLine, const EnvProvider& env) { - return parseTokens (preprocess (commandLine)).settings; + return parseTokens (preprocess (commandLine), env).settings; } //============================================================================== diff --git a/Source/SettingsParser.h b/Source/SettingsParser.h index 09cabe4..3cd6ebc 100644 --- a/Source/SettingsParser.h +++ b/Source/SettingsParser.h @@ -17,17 +17,26 @@ #include "PluginvalSettings.h" #include "PluginTests.h" +#include + /** The command-line -> settings pipeline. - CLI11 binds each option directly to a member of a single PluginvalSettings - instance (environment variables via ->envname() on the same line). A - --config JSON file seeds the struct before parsing so precedence is - defaults < config < env < CLI. Conversion to the JUCE-flavoured + A single PluginvalSettings is filled by successive layers, lowest to highest + precedence: hardcoded defaults, then environment variables, then --config + (repeatable JSON files, later files win per key), then the individual CLI options. + CLI11 binds each option to a member and also provides the coercion used for + the environment layer (env names are derived from the registered options, so + there is no separate env table). Conversion to the JUCE-flavoured PluginTests::Options happens at the boundary via toPluginTestOptions(). */ namespace settings_parser { + //============================================================================== + /** Returns the value of an environment variable, or "" if unset. */ + using EnvProvider = std::function; + juce::String systemEnv (const juce::String& name); + //============================================================================== /** Tokenises + preprocesses a raw command line: rewrites the deprecated "strictnessLevel", strips the macOS "-NSDocumentRevisionsDebugMode YES" @@ -54,10 +63,10 @@ namespace settings_parser }; /** Parses preprocessed option tokens into settings (and handles --help/--version). */ - ParseResult parseTokens (const juce::StringArray& tokens); + ParseResult parseTokens (const juce::StringArray& tokens, const EnvProvider& env = systemEnv); /** Convenience: preprocess + parse from a raw command line, returning settings. */ - PluginvalSettings parse (const juce::String& commandLine); + PluginvalSettings parse (const juce::String& commandLine, const EnvProvider& env = systemEnv); //============================================================================== /** Serialises options for the child validation process as a base64 JSON diff --git a/docs/Command line options.md b/docs/Command line options.md index 1267c53..4c82469 100644 --- a/docs/Command line options.md +++ b/docs/Command line options.md @@ -8,38 +8,32 @@ pluginval [OPTIONS] OPTIONS: -h, --help Print this help message and exit --version Display program version information and exit - --config TEXT Path to a JSON settings file (overridden by env vars and CLI - options). + --config TEXT ... Path to a JSON settings file. Repeatable; later files win per + key. --validate TEXT Validates the plugin at the given path (or AU id). - --strictness-level INT (Env:STRICTNESS_LEVEL) + --strictness-level INT Strictness level 1-10 (default 5). - --timeout-ms INT (Env:TIMEOUT_MS) - Test timeout in ms (default 30000, -1 to never timeout). - --repeat INT (Env:REPEAT) - Number of times to repeat the tests. - --randomise (Env:RANDOMISE) - Run the tests in a random order per repeat. - --verbose (Env:VERBOSE) - Output additional logging information. - --skip-gui-tests (Env:SKIP_GUI_TESTS) - Avoid tests that create GUI windows (for headless CI). - --sample-rates FLOAT ... (Env:SAMPLE_RATES) + --timeout-ms INT Test timeout in ms (default 30000, -1 to never timeout). + --repeat INT Number of times to repeat the tests. + --randomise Run the tests in a random order per repeat. + --verbose Output additional logging information. + --skip-gui-tests Avoid tests that create GUI windows (for headless CI). + --sample-rates FLOAT ... Comma-separated sample rates (default 44100,48000,96000). - --block-sizes INT ... (Env:BLOCK_SIZES) + --block-sizes INT ... Comma-separated block sizes (default 64,128,256,512,1024). - --data-file TEXT (Env:DATA_FILE) - Path to a data file tests can use to configure themselves. - --output-dir TEXT (Env:OUTPUT_DIR) - Directory in which to write the log files. - --output-filename TEXT (Env:OUTPUT_FILENAME) + --data-file TEXT Path to a data file tests can use to configure themselves. + --output-dir TEXT Directory in which to write the log files. + --output-filename TEXT Filename to write logs into. --disabled-tests TEXT Comma-separated test names, or a path to a file listing them. - --random-seed TEXT (Env:RANDOM_SEED) - Random seed (hex 0x.. or int) for replicable test runs. - --rtcheck ENUM:value in {disabled->0,enabled->1,relaxed->2} OR {0,1,2} (Env:RTCHECK) + --random-seed TEXT Random seed (hex 0x.. or int) for replicable test runs. + --rtcheck ENUM:value in {disabled->0,enabled->1,relaxed->2} OR {0,1,2} Real-time safety checks: disabled, enabled or relaxed. +JUCE v8.0.13 + Other commands: --run-tests Run the internal unit tests. --strictness-help [level] List all tests that run at the given strictness level. @@ -53,4 +47,6 @@ prefix dashes, converting internal dashes to underscores and capitalising, e.g. "--skip-gui-tests" -> "SKIP_GUI_TESTS=1" "--timeout-ms 30000" -> "TIMEOUT_MS=30000" -Precedence: command-line options > environment variables > --config file. +Precedence (lowest to highest): defaults, environment variables, --config, +command-line options. +--config is repeatable; later files win per key. From 3baec446c91fd2cf1d72f6f991a21df98c60327b Mon Sep 17 00:00:00 2001 From: David Rowland Date: Sun, 7 Jun 2026 13:31:23 +0100 Subject: [PATCH 09/12] Add 2.0.0 changelist, harden --config-base64, add subcommands handoff - CHANGELIST.md: 2.0.0 entry summarising the CLI11/JSON parser rewrite, --config, the new precedence, and the breaking changes (VERSION not bumped). - --config-base64 (internal parent->child handoff) now errors if combined with any option other than --validate, instead of silently ignoring it. Test added. - SUBCOMMANDS_HANDOFF.md: handoff doc for the deferred subcommand restructure (validate/run-tests/strictness-help with deprecated flat-flag aliases). Co-Authored-By: Claude Opus 4.8 --- CHANGELIST.md | 10 +++++ SUBCOMMANDS_HANDOFF.md | 89 +++++++++++++++++++++++++++++++++++++ Source/CommandLineTests.cpp | 24 ++++++++++ Source/SettingsParser.cpp | 21 ++++++++- 4 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 SUBCOMMANDS_HANDOFF.md diff --git a/CHANGELIST.md b/CHANGELIST.md index a97b26a..cfa82b3 100644 --- a/CHANGELIST.md +++ b/CHANGELIST.md @@ -1,5 +1,15 @@ # pluginval Change List +### 2.0.0 +- Replaced the hand-rolled command-line parser with CLI11 and a single JSON-based settings pipeline +- Added `--config ` to load settings from JSON (repeatable; later files win per key) +- Settings precedence is now (lowest to highest): defaults, environment variables, `--config`, command-line options +- Environment variable support is now automatic for every option (including `DISABLED_TESTS`) +- **Breaking:** unrecognised options now produce an error instead of being ignored +- **Breaking:** invalid option values now error instead of silently falling back (e.g. `--rtcheck banana`, `--strictness-level abc`) +- **Breaking:** falsey flag environment variables now mean off (e.g. `SKIP_GUI_TESTS=0` no longer enables skipping) +- `--help` output is now generated by CLI11 + ### 1.0.5 - Added static linking to the Windows runtime so it should run on more Windows systems (particularly non-dev machines) - Made `PluginInfoTest` run on the message thread as the functions it calls aren't thread-safe diff --git a/SUBCOMMANDS_HANDOFF.md b/SUBCOMMANDS_HANDOFF.md new file mode 100644 index 0000000..3f6ef27 --- /dev/null +++ b/SUBCOMMANDS_HANDOFF.md @@ -0,0 +1,89 @@ +# HANDOFF: Restructure pluginval's CLI into subcommands + +## Goal +Turn the current "mode" flags into proper subcommands, each with its own +argument set, while keeping the old flat flags working as **deprecated aliases +for one release**: + +| New (target) | Old (keep as deprecated alias) | +|---|---| +| `pluginval validate [options] ` (the **default**) | `pluginval --validate ` / `pluginval ` | +| `pluginval run-tests` | `pluginval --run-tests` | +| `pluginval strictness-help [level]` | `pluginval --strictness-help [level]` | +| `pluginval --version` / `pluginval --help` | (unchanged) | + +All settings options (`--strictness-level`, `--config`, `--rtcheck`, …), +environment variables, and the JSON precedence pipeline belong to the `validate` +subcommand and are otherwise unchanged. + +This was deliberately deferred from the CLI11 refactor PR (#174). The pipeline +and `PluginvalSettings` were designed to be reused unchanged. + +## Read these first (don't trust this summary — verify against the files) +- `Source/SettingsParser.cpp/.h` — the parser. Key pieces to reuse: + - `preprocess()` — deprecation rewrite, macOS flag strip, implicit `--validate`, tokenise. + - `parseTokens(tokens, env)` — env → `--config` → CLI layering into one `PluginvalSettings`. **This is the `validate` body.** + - `configureApp(app, s)` — registers every option on a `CLI::App`, used for both the env and CLI passes. + - `createChildProcessCommandLine()` — the parent→child base64 handoff (`--config-base64 --validate `). + - `isCommandLine(tokens)` — what makes `shouldPerformCommandLine` return true. +- `Source/CommandLine.cpp` — `performCommandLine()` is the dispatcher today: + token-scans for `--run-tests` / `--strictness-help`, otherwise runs `parseTokens` for validate; `--help`/`--version` go through CLI11. Also `runUnitTests()`, `printStrictnessHelp()`. +- `Source/Main.cpp` — calls `shouldPerformCommandLine()` then `performCommandLine()` with `getCommandLineParameters()` (a single `juce::String`). +- `Source/CommandLineTests.cpp` — the test contract. + +## Recommended approach: a thin `argv[1]` verb dispatcher (not CLI11 subcommands) +The existing `parseTokens` pipeline (env/config/CLI layering via two `CLI::App` +passes) is the hard part and already works. Rather than re-express it inside +CLI11's native `add_subcommand` machinery (which complicates the env/config +layering because the options live on the subcommand), peel the verb off the +front and route: + +1. In a new `dispatch()` step (in `SettingsParser` or `CommandLine.cpp`): + - Tokenise the command line (reuse `preprocess` minus the implicit-validate step, or add a pre-step). + - Look at the first non-option token: + - `validate` → strip it, run the existing validate pipeline on the rest. + - `run-tests` → `runUnitTests()`. + - `strictness-help` → `printStrictnessHelp(level)`. + - otherwise → **default to validate** (this preserves `pluginval ` and `pluginval --strictness-level 5 `). +2. Each verb keeps its own small set of expected args. `validate` reuses + `configureApp`/`parseTokens` verbatim. `run-tests` takes none. + `strictness-help` takes an optional level. + +This keeps `parseTokens` and the precedence layering untouched — the subcommand +work is purely a routing layer in front of it. + +(If you prefer CLI11-native subcommands instead: put `configureApp` options on a +`validate` subcommand, and make `buildEnvArgv`/the env pass target that +subcommand's options. Doable, but more invasive for no functional gain here.) + +## Deprecated-alias behaviour (one release) +Keep the old flat forms working, but print a one-line notice to stderr pointing +at the new syntax, e.g.: +- `pluginval --validate x` → run validate; warn `"--validate is deprecated; use 'pluginval validate x'"`. +- `pluginval --run-tests` → warn `"use 'pluginval run-tests'"`. +- `pluginval --strictness-help` → warn `"use 'pluginval strictness-help'"`. +- `pluginval ` (bare path) → **no warning** (still the documented shorthand for `validate`). + +Gate the warnings behind detection of the old flag so the new subcommand form is +silent. Remove the aliases in the release after next; note it in `CHANGELIST.md`. + +## Files to touch +- `Source/SettingsParser.{h,cpp}` — add the verb dispatch + a `Command` result (validate/run-tests/strictness-help/help/version), or expose a `dispatch()` that returns which verb + the remaining tokens. Keep `parseTokens` as the validate body. +- `Source/CommandLine.cpp` — `performCommandLine()` routes on the verb; emit the deprecation notices for old flat flags. `shouldPerformCommandLine()` must also recognise the bare verbs (`validate`/`run-tests`/`strictness-help`) in addition to the old flags. +- `Source/CommandLineTests.cpp` — add tests: each subcommand; default-to-validate; bare-path shorthand; every deprecated alias still works (and warns); `run-tests`/`strictness-help` arg handling. +- `docs/Command line options.md` — regenerate (`pluginval --help`); CLI11 can show per-subcommand help if you go native, otherwise hand-format the verb list. +- `CHANGELIST.md` — note the subcommand syntax + the deprecation. +- `CLAUDE.md` — update the "CLI Settings Pipeline" section to mention the verb layer. + +## Gotchas / decisions to make +- **Child-process handoff.** `createChildProcessCommandLine()` emits `--config-base64 --validate `. Decide whether the child invocation becomes `validate --config-base64 …` or stays flat. Simplest: keep it flat and have the dispatcher treat a leading `--config-base64`/`--validate` as the (deprecated, unwarned-for-internal) validate path. Note `--config-base64` is now hardened to reject being combined with non-`--validate` options — keep that working under whichever form you choose. +- **`shouldPerformCommandLine`** is what flips pluginval into CLI (vs GUI) mode in `Main.cpp`. It must return true for `pluginval run-tests` etc., not just the old flags. +- **`--help` scope.** With the dispatcher, `pluginval --help` is the top-level help (list verbs + the validate options). Consider `pluginval validate --help` for the full option list. CLI11-native subcommands give this for free. +- **Implicit validate** currently lives in `preprocess`. With an explicit `validate` verb, make sure `pluginval ` (no verb) still resolves to validate, and `pluginval validate ` doesn't double-insert `--validate`. +- **Reconcile** a positional plugin path under `validate` (e.g. `pluginval validate `) with the existing `--validate ` option — pick one canonical form (recommend the positional for the new syntax, mapping it onto `s.validatePath`). + +## Verify +- `pluginval run-tests` passes the full unit suite (it must, it's how CI runs tests). +- `pluginval validate --strictness-level 10 ` and `pluginval ` both validate. +- Every deprecated alias produces identical behaviour to before (plus a notice). +- CI matrix green (Linux/macOS/Windows build + dependency). Remember `.github/workflows/build.yaml` uses `--run-tests` and `--strictness-level 10 --validate …`; update those to the new syntax **and** keep an alias test, or the deprecation will fire in CI. diff --git a/Source/CommandLineTests.cpp b/Source/CommandLineTests.cpp index 24f3d14..e537a29 100644 --- a/Source/CommandLineTests.cpp +++ b/Source/CommandLineTests.cpp @@ -307,6 +307,30 @@ struct CommandLineTests : public juce::UnitTest expect (PluginvalSettings::fromPluginTestOptions (opts2, fileOrID2) == expected); } + + beginTest ("--config-base64 rejects extra options"); + { + PluginTests::Options opts; + opts.strictnessLevel = 7; + + juce::StringArray childArgs (createCommandLine ("/some/MyPlugin.vst3", opts)); + childArgs.remove (0); // drop the executable path + + // The legitimate parent -> child handoff parses fine. + { + const auto r = settings_parser::parseTokens (childArgs, emptyEnv()); + expect (! r.handled); + expectEquals (r.settings.strictnessLevel, 7); + } + + // Combining it with any other option is rejected. + { + childArgs.addArray ({ "--strictness-level", "9" }); + const auto r = settings_parser::parseTokens (childArgs, emptyEnv()); + expect (r.handled); + expectEquals (r.exitCode, 1); + } + } } }; diff --git a/Source/SettingsParser.cpp b/Source/SettingsParser.cpp index 6d302cc..244286a 100644 --- a/Source/SettingsParser.cpp +++ b/Source/SettingsParser.cpp @@ -316,9 +316,28 @@ Precedence (lowest to highest): defaults, environment variables, --config, comma ParseResult result; auto& s = result.settings; - // A base64 JSON handoff from the parent process is fully authoritative. + // --config-base64 carries a fully-resolved settings set from the parent + // process and is authoritative. It is for internal use only and must not be + // combined with other options (which would otherwise be silently ignored); + // the only companion allowed is --validate. if (const auto b64 = valueForOption (tokens, "--config-base64"); b64.isNotEmpty()) { + for (const auto& token : tokens) + { + if (! token.startsWith ("-") || token == "-") + continue; // a value, not an option + + if (const auto name = token.upToFirstOccurrenceOf ("=", false, false); + name != "--config-base64" && name != "--validate") + { + std::cerr << "*** FAILED: --config-base64 is for internal use and cannot be combined " + << "with other options (got " << name << ")" << std::endl; + result.exitCode = 1; + result.handled = true; + return result; + } + } + s = settings_serializer::fromJsonString (decodeBase64 (b64).toStdString()); s.validatePath = resolvePluginPath (juce::String (s.validatePath)).toStdString(); return result; From 6f6c33a8bb86017c25d9798f1c27e72917630dea Mon Sep 17 00:00:00 2001 From: David Rowland Date: Sun, 7 Jun 2026 13:42:31 +0100 Subject: [PATCH 10/12] Audit CLAUDE.md: remove non-existent --validate-in-process CLI flag - --validate-in-process is not a CLI option (launchInProcess is GUI/internal only); the new strict parser would now error on it. Removed from the options list and corrected the process-model notes (GUI uses child processes; the CLI --validate runs in-process with signal-handler crash reporting). - Note the staged 2.0.0 CHANGELIST entry vs the un-bumped VERSION. Co-Authored-By: Claude Opus 4.8 --- CLAUDE.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index f466f3f..4acbea1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ **pluginval** is a cross-platform audio plugin validator and tester application developed by Tracktion Corporation. It tests VST, VST3, AU (Audio Unit), LV2, and LADSPA plugins for compatibility and stability with host applications. -- **Version**: 1.0.4 (see `VERSION` file) +- **Version**: 1.0.4 (see `VERSION` file; a 2.0.0 entry is staged in `CHANGELIST.md` but `VERSION` is not yet bumped) - **License**: GPLv3 - **Framework**: Built on JUCE (v8.0.x) - **Language**: C++20 @@ -368,7 +368,6 @@ Key options: - `--config [file.json]` - Load a full settings set from JSON (overridden by env vars and CLI options) - `--strictness-level [1-10]` - Test thoroughness (default: 5) - `--skip-gui-tests` - Skip GUI tests (for headless CI) -- `--validate-in-process` - Don't use child process (for debugging) - `--timeout-ms [ms]` - Test timeout (default: 30000, -1 for none) - `--verbose` - Enable verbose logging - `--output-dir [dir]` - Directory for log files @@ -480,6 +479,6 @@ Run internal tests via CLI: - Always test changes on multiple platforms when possible - VST3 plugins have specific threading requirements - use the `*OnMessageThreadIfVST3` helpers -- Child process validation is the default and recommended for production use -- In-process validation (`--validate-in-process`) is useful for debugging but a crashing plugin will crash pluginval +- The GUI runs each validation in a separate child process for crash isolation (the default) +- The CLI `--validate` path runs in-process; a crashing plugin will terminate pluginval, and the signal handler reports it as a failure rather than a pass - Real-time safety checking is only available on macOS currently (uses rtcheck library) From df1179f87a9a91479945a43c459c46fe6b33786e Mon Sep 17 00:00:00 2001 From: David Rowland Date: Sun, 7 Jun 2026 13:58:39 +0100 Subject: [PATCH 11/12] Compile EditorTests and ExtremeTests (were present but not built) Source/tests/EditorTests.cpp and Source/tests/ExtremeTests.cpp were never added to SourceFiles, so their self-registering PluginTest instances were never compiled in and the tests never ran. Add them to the build. Re-enables: "Editor stress" (L6), "Editor DPI Awareness" (L3, Windows only), "Allocations during process" (L9), "Process called with a larger than prepared block size". Verified they compile and pass a real strictness-10 validation. Co-Authored-By: Claude Opus 4.8 --- CHANGELIST.md | 1 + CMakeLists.txt | 2 ++ 2 files changed, 3 insertions(+) diff --git a/CHANGELIST.md b/CHANGELIST.md index cfa82b3..79d4c75 100644 --- a/CHANGELIST.md +++ b/CHANGELIST.md @@ -9,6 +9,7 @@ - **Breaking:** invalid option values now error instead of silently falling back (e.g. `--rtcheck banana`, `--strictness-level abc`) - **Breaking:** falsey flag environment variables now mean off (e.g. `SKIP_GUI_TESTS=0` no longer enables skipping) - `--help` output is now generated by CLI11 +- Enabled the editor stress and extreme (real-time allocation / oversized block) tests that were present in the source tree but had not been compiled into the build ### 1.0.5 - Added static linking to the Windows runtime so it should run on more Windows systems (particularly non-dev machines) diff --git a/CMakeLists.txt b/CMakeLists.txt index 180d6ef..20e02b6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -137,6 +137,8 @@ set(SourceFiles Source/tests/BasicTests.cpp Source/tests/LocaleTest.cpp Source/tests/BusTests.cpp + Source/tests/EditorTests.cpp + Source/tests/ExtremeTests.cpp Source/tests/ParameterFuzzTests.cpp Source/TestUtilities.cpp Source/Validator.cpp) From 62793791665cdd7c5e6e38312422401d48bad001 Mon Sep 17 00:00:00 2001 From: David Rowland Date: Sun, 7 Jun 2026 14:11:07 +0100 Subject: [PATCH 12/12] Fix Windows-only compile error in EditorDPITest Qualify juce::ScopedDPIAwarenessDisabler. The EditorDPITest block is guarded by #if JUCE_WINDOWS && JUCE_WIN_PER_MONITOR_DPI_AWARE, so this latent bug only surfaced now that EditorTests.cpp is compiled (and only on Windows). Co-Authored-By: Claude Opus 4.8 --- Source/tests/EditorTests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/tests/EditorTests.cpp b/Source/tests/EditorTests.cpp index 25fc7c4..8eb6634 100644 --- a/Source/tests/EditorTests.cpp +++ b/Source/tests/EditorTests.cpp @@ -63,7 +63,7 @@ struct EditorDPITest : public PluginTest { if (instance.hasEditor()) { - ScopedDPIAwarenessDisabler scopedDPIAwarenessDisabler; + juce::ScopedDPIAwarenessDisabler scopedDPIAwarenessDisabler; { ut.logMessage ("Testing opening Editor with DPI Awareness disabled");