Add cli parser to the test runner

.github/workflows/ci-cpp.yml

@@ -79,7 +79,7 @@ jobs:
           SPICE_STD_DIR: /home/runner/work/spice/spice/std
         run: |
           cd ./bin/test
-          ./spicetest false
+          ./spicetest --skip-github-tests
 
       - name: Generate coverage report
         run: |

.run/spicetest.run.xml

@@ -1,5 +1,5 @@
 <component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="spicetest" type="CMakeGoogleTestRunConfigurationType" factoryName="Google Test" PROGRAM_PARAMS="false" REDIRECT_INPUT="false" ELEVATE="false" USE_EXTERNAL_CONSOLE="false" PASS_PARENT_ENVS_2="true" PROJECT_NAME="Spice" TARGET_NAME="spicetest" CONFIG_NAME="Debug" RUN_TARGET_PROJECT_NAME="Spice" RUN_TARGET_NAME="spicetest" TEST_CLASS="AnalyzerTests" TEST_MODE="SUITE_TEST">
+  <configuration default="false" name="spicetest" type="CMakeGoogleTestRunConfigurationType" factoryName="Google Test" PROGRAM_PARAMS="--update-refs=false" REDIRECT_INPUT="false" ELEVATE="false" USE_EXTERNAL_CONSOLE="false" PASS_PARENT_ENVS_2="true" PROJECT_NAME="Spice" TARGET_NAME="spicetest" CONFIG_NAME="Debug" RUN_TARGET_PROJECT_NAME="Spice" RUN_TARGET_NAME="spicetest" TEST_CLASS="AnalyzerTests" TEST_MODE="SUITE_TEST">
     <envs>
       <env name="RUN_TESTS" value="ON" />
       <env name="SPICE_STD_DIR" value="$PROJECT_DIR$/std" />

src/cli/CLIInterface.cpp

@@ -279,8 +279,8 @@ void CLIInterface::addCompileSubcommandOptions(CLI::App *subCmd) {
 int CLIInterface::parse(int argc, char **argv) {
   try {
     app.parse(argc, argv);
-  } catch (const CLI::ParseError &e) {
-    return app.exit(e);
+  } catch (const CLI::ParseError &parseError) {
+    return app.exit(parseError);
   }
   return 0;
 }

src/cli/CLIInterface.h

@@ -63,5 +63,5 @@ class CLIInterface {
   void addCompileSubcommandOptions(CLI::App *subCmd);
 
   // Members
-  CLI::App app = CLI::App{"Spice Programming Language", "Spice"};
+  CLI::App app = CLI::App{"Spice Programming Language", "spice"};
 };

src/symboltablebuilder/SymbolTableBuilder.cpp

@@ -57,7 +57,7 @@ std::any SymbolTableBuilder::visitMainFctDef(MainFctDefNode *node) {
 
 std::any SymbolTableBuilder::visitFctDef(FctDefNode *node) {
   // Build function specifiers
-  auto specifiers = SymbolSpecifiers::of(TY_FUNCTION);
+  SymbolSpecifiers specifiers = SymbolSpecifiers::of(TY_FUNCTION);
   if (SpecifierLstNode *specifierLst = node->specifierLst(); specifierLst) {
     for (const SpecifierNode *specifier : specifierLst->specifiers()) {
       if (specifier->type == SpecifierNode::TY_INLINE)
@@ -120,7 +120,7 @@ std::any SymbolTableBuilder::visitFctDef(FctDefNode *node) {
 
 std::any SymbolTableBuilder::visitProcDef(ProcDefNode *node) {
   // Build procedure specifiers
-  auto specifiers = SymbolSpecifiers::of(TY_PROCEDURE);
+  SymbolSpecifiers specifiers = SymbolSpecifiers::of(TY_PROCEDURE);
   if (SpecifierLstNode *specifierLst = node->specifierLst(); specifierLst) {
     for (const SpecifierNode *specifier : specifierLst->specifiers()) {
       if (specifier->type == SpecifierNode::TY_INLINE)

test/CMakeLists.txt

@@ -5,8 +5,8 @@ antlr_target(Spice ${CMAKE_CURRENT_SOURCE_DIR}/../src/Spice.g4 VISITOR)
 set(SOURCES
         main.cpp
         TestRunner.cpp
-        TestUtil.cpp
-        TestUtil.h)
+        util/TestUtil.cpp
+        util/TestUtil.h cli/CLIInterface.cpp cli/CLIInterface.h)
 
 add_executable(spicetest ${SOURCES} ${ANTLR_Spice_CXX_OUTPUTS})
 

test/TestRunner.cpp

@@ -9,8 +9,8 @@
 #include <llvm/ADT/Triple.h>
 #include <llvm/Support/Host.h>
 
-#include "TestUtil.h"
 #include "symboltablebuilder/SymbolTable.h"
+#include "util/TestUtil.h"
 #include <SourceFile.h>
 #include <cli/CLIInterface.h>
 #include <exception/LexerError.h>

test/cli/CLIInterface.cpp

@@ -0,0 +1,46 @@
+// Copyright (c) 2021-2022 ChilliBits. All rights reserved.
+
+#include "CLIInterface.h"
+
+// GCOV_EXCL_START
+
+void CLIInterface::createInterface() {
+  // Allow positional args
+  app.allow_windows_style_options();
+  app.allow_extras();
+  app.positionals_at_end();
+  app.footer("(c) Marc Auberer 2021-2022");
+
+  // Add version flag
+  std::string versionName = std::string(SPICE_VERSION);
+  std::string builtBy = std::string(SPICE_BUILT_BY);
+  std::string versionString = "Spice version " + versionName + "\nbuilt by: " + builtBy + "\n\n(c) Marc Auberer 2021-2022";
+  app.set_version_flag("--version,-v", versionString);
+}
+
+void CLIInterface::addOptions(bool &updateRefs, bool &runBenchmarks, bool &skipNonGitHubTests) {
+  // --update-refs
+  app.add_flag<bool>("--update-refs,-u", updateRefs, "Update test reference files");
+  // --run-benchmarks
+  app.add_flag<bool>("--run-benchmarks,-b", runBenchmarks, "Also run benchmarks and check baseline values");
+  // --skip-github-tests
+  app.add_flag<bool>("--skip-github-tests,-gh", skipNonGitHubTests, "Skip non-working tests on GitHub Actions");
+}
+
+/**
+ * Start the parsing process
+ *
+ * @param argc Argument count
+ * @param argv Argument vector
+ * @return Return code
+ */
+int CLIInterface::parse(int argc, char **argv) {
+  try {
+    app.parse(argc, argv);
+  } catch (const CLI::ParseError &parseError) {
+    return app.exit(parseError);
+  }
+  return 0;
+}
+
+// GCOV_EXCL_STOP

test/cli/CLIInterface.h

@@ -0,0 +1,27 @@
+// Copyright (c) 2021-2022 ChilliBits. All rights reserved.
+
+#pragma once
+
+#include "../../lib/cli11/CLI11.hpp"
+
+// GCOV_EXCL_START
+
+/**
+ * Helper class to setup the cli interface and command line parser
+ */
+class CLIInterface {
+public:
+  // Constructors
+  explicit CLIInterface() = default;
+
+  // Public methods
+  void createInterface();
+  void addOptions(bool &updateRefs, bool &runBenchmarks, bool &skipNonGitHubTests);
+  int parse(int argc, char **argv);
+
+private:
+  // Private members
+  CLI::App app = CLI::App{"Spice Test Runner", "spice"};
+};
+
+// GCOV_EXCL_STOP

test/main.cpp

@@ -1,10 +1,14 @@
 // Copyright (c) 2021-2022 ChilliBits. All rights reserved.
 
-// GCOV_EXCL_START
+#include "cli/CLIInterface.h"
 
 #include <gtest/gtest.h>
 
+// GCOV_EXCL_START
+
 bool updateRefs = false;
+bool runBenchmarks = false;
+bool skipNonGitHubTests = false;
 
 /**
  * Entry point to the Spice testing suite
@@ -13,16 +17,15 @@ bool updateRefs = false;
  * @param argv Argument vector
  * @return Return code
  */
-int main(int argc, char **argv) { // Call ./spicetest <true|false>
-  // Parse cli args
-  std::vector<std::string> args;
-  for (size_t i = 1; i < argc; i++)
-    args.emplace_back(argv[i]);
-
-  // Extract cli args
-  updateRefs = !args.empty() && args[0] == "true";
-
+int main(int argc, char **argv) {
   testing::InitGoogleTest(&argc, argv);
+  // Initialize command line parser
+  CLIInterface cli;
+  cli.createInterface();
+  cli.addOptions(updateRefs, runBenchmarks, skipNonGitHubTests);
+  // Parse command line args
+  cli.parse(argc, argv);
+  // Run tests
   return RUN_ALL_TESTS();
 }
 

test/TestUtil.cpp → test/util/TestUtil.cpp

@@ -8,8 +8,8 @@
 
 #include <gtest/gtest.h>
 
-#include <util/CommonUtil.h>
-#include <util/FileUtil.h>
+#include "util/CommonUtil.h"
+#include "util/FileUtil.h"
 
 #ifdef OS_UNIX
 #include <cstring> // Required by builds on Linux

test/TestUtil.h → test/util/TestUtil.h

@@ -10,7 +10,7 @@
 
 #include <gtest/gtest.h>
 
-#include <util/FileUtil.h>
+#include "util/FileUtil.h"
 
 #if OS_WINDOWS
 const char *const PATH_TEST_FILES = ".\\test-files\\";