diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 07902e917..9a45871af 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -246,6 +246,18 @@ jobs: chmod u+x ./lib/modules/.github/run-tests (./lib/modules/.github/run-tests src) + - uses: actions/setup-python@v5 + if: ${{ ! startsWith(matrix.config.name, 'Windows') }} + with: + python-version: '3.13' + + - name: REPL tests + if: ${{ ! startsWith(matrix.config.name, 'Windows') }} + shell: bash + run: | + pip install -r tests/repl/requirements.txt + python3 tests/repl/test.py + fuzzing: runs-on: ubuntu-24.04 name: Fuzz testing diff --git a/CMakeLists.txt b/CMakeLists.txt index 91828b50e..c77227f3a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -167,17 +167,22 @@ if (ARK_BUILD_MODULES) set_directory_properties(PROPERTIES COMPILE_OPTIONS "${old_dir_compile_options}") endif () +if (ARK_TESTS OR ARK_BUILD_EXE) + add_subdirectory("${ark_SOURCE_DIR}/lib/replxx" EXCLUDE_FROM_ALL) +endif () + if (ARK_TESTS) - file(GLOB_RECURSE UT_SOURCES + file(GLOB_RECURSE SOURCES ${ark_SOURCE_DIR}/tests/unittests/*.cpp ${ark_SOURCE_DIR}/lib/fmt/src/format.cc ${ark_SOURCE_DIR}/src/arkscript/Formatter.cpp - ${ark_SOURCE_DIR}/src/arkscript/JsonCompiler.cpp) - add_executable(unittests ${UT_SOURCES}) + ${ark_SOURCE_DIR}/src/arkscript/JsonCompiler.cpp + ${ark_SOURCE_DIR}/src/arkscript/REPL/Utils.cpp) + add_executable(unittests ${SOURCES}) add_subdirectory(${ark_SOURCE_DIR}/lib/ut) target_include_directories(unittests PUBLIC ${ark_SOURCE_DIR}/include) - target_link_libraries(unittests PUBLIC ArkReactor ut) + target_link_libraries(unittests PUBLIC ArkReactor ut replxx) add_compile_definitions(BOOST_UT_DISABLE_MODULE) target_compile_features(unittests PRIVATE cxx_std_20) @@ -233,7 +238,6 @@ if (ARK_BUILD_EXE) ${ark_SOURCE_DIR}/lib/fmt/src/format.cc) add_executable(arkscript ${EXE_SOURCES}) - add_subdirectory("${ark_SOURCE_DIR}/lib/replxx" EXCLUDE_FROM_ALL) add_subdirectory("${ark_SOURCE_DIR}/lib/clipp" EXCLUDE_FROM_ALL) target_include_directories(arkscript SYSTEM PUBLIC "${ark_SOURCE_DIR}/lib/clipp/include") diff --git a/include/CLI/REPL/Utils.hpp b/include/CLI/REPL/Utils.hpp index 461ef004f..7004f3a68 100644 --- a/include/CLI/REPL/Utils.hpp +++ b/include/CLI/REPL/Utils.hpp @@ -34,6 +34,19 @@ namespace Ark::internal */ void trimWhitespace(std::string& line); + /** + * @brief Compute a list of all the language keywords and builtins + * + * @return std::vector + */ + std::vector getAllKeywords(); + + /** + * @brief Compute a list of pairs (word -> color) to be used for coloration by the REPL + * @return std::vector> + */ + std::vector> getColorPerKeyword(); + replxx::Replxx::completions_t hookCompletion(const std::vector& words, const std::string& context, int& length); void hookColor(const std::vector>& words_colors, const std::string& context, replxx::Replxx::colors_t& colors); diff --git a/src/arkscript/REPL/Repl.cpp b/src/arkscript/REPL/Repl.cpp index 609fcefca..fbd271b64 100644 --- a/src/arkscript/REPL/Repl.cpp +++ b/src/arkscript/REPL/Repl.cpp @@ -4,7 +4,6 @@ #include #include -#include #include #include @@ -19,37 +18,8 @@ namespace Ark m_old_ip(0), m_lib_env(lib_env), m_state(m_lib_env), m_vm(m_state), m_has_init_vm(false) { - m_keywords.reserve(keywords.size() + Language::listInstructions.size() + Language::operators.size() + Builtins::builtins.size() + 2); - for (auto keyword : keywords) - m_keywords.emplace_back(keyword); - for (auto inst : Language::listInstructions) - m_keywords.emplace_back(inst); - for (auto op : Language::operators) - m_keywords.emplace_back(op); - for (const auto& builtin : std::ranges::views::keys(Builtins::builtins)) - m_keywords.push_back(builtin); - m_keywords.emplace_back("and"); - m_keywords.emplace_back("or"); - - m_words_colors.reserve(keywords.size() + Language::listInstructions.size() + Language::operators.size() + Builtins::builtins.size() + 4); - for (auto keyword : keywords) - m_words_colors.emplace_back(keyword, Replxx::Color::BRIGHTRED); - for (auto inst : Language::listInstructions) - m_words_colors.emplace_back(inst, Replxx::Color::GREEN); - for (auto op : Language::operators) - { - auto safe_op = std::string(op); - if (const auto it = safe_op.find_first_of(R"(-+=/*<>[]()?")"); it != std::string::npos) - safe_op.insert(it, "\\"); - m_words_colors.emplace_back(safe_op, Replxx::Color::BRIGHTBLUE); - } - for (const auto& builtin : std::ranges::views::keys(Builtins::builtins)) - m_words_colors.emplace_back(builtin, Replxx::Color::GREEN); - - m_words_colors.emplace_back("and", Replxx::Color::BRIGHTBLUE); - m_words_colors.emplace_back("or", Replxx::Color::BRIGHTBLUE); - m_words_colors.emplace_back("[\\-|+]?[0-9]+(\\.[0-9]+)?", Replxx::Color::YELLOW); - m_words_colors.emplace_back("\".*\"", Replxx::Color::MAGENTA); + m_keywords = getAllKeywords(); + m_words_colors = getColorPerKeyword(); } int Repl::run() @@ -59,7 +29,7 @@ namespace Ark while (m_running) { - auto maybe_block = getCodeBlock(); + std::optional maybe_block = getCodeBlock(); // save a valid ip if execution failed m_old_ip = m_vm.m_execution_contexts[0]->ip; diff --git a/src/arkscript/REPL/Utils.cpp b/src/arkscript/REPL/Utils.cpp index e6e5f806f..814b36b32 100644 --- a/src/arkscript/REPL/Utils.cpp +++ b/src/arkscript/REPL/Utils.cpp @@ -2,6 +2,11 @@ #include #include +#include +#include + +#include +#include namespace Ark::internal { @@ -20,27 +25,73 @@ namespace Ark::internal } } + std::vector getAllKeywords() + { + std::vector output; + output.reserve(keywords.size() + Language::listInstructions.size() + Language::operators.size() + Builtins::builtins.size() + 2); + for (auto keyword : keywords) + output.emplace_back(keyword); + for (auto inst : Language::listInstructions) + output.emplace_back(inst); + for (auto op : Language::operators) + output.emplace_back(op); + for (const auto& builtin : std::ranges::views::keys(Builtins::builtins)) + output.push_back(builtin); + output.emplace_back("and"); + output.emplace_back("or"); + + return output; + } + + std::vector> getColorPerKeyword() + { + using namespace replxx; + + std::vector> output; + output.reserve(keywords.size() + Language::listInstructions.size() + Language::operators.size() + Builtins::builtins.size() + 4); + for (auto keyword : keywords) + output.emplace_back(keyword, Replxx::Color::BRIGHTRED); + for (auto inst : Language::listInstructions) + output.emplace_back(inst, Replxx::Color::GREEN); + for (auto op : Language::operators) + { + auto safe_op = std::string(op); + if (const auto it = safe_op.find_first_of(R"(-+=/*<>[]()?")"); it != std::string::npos) + safe_op.insert(it, "\\"); + output.emplace_back(safe_op, Replxx::Color::BRIGHTBLUE); + } + for (const auto& builtin : std::ranges::views::keys(Builtins::builtins)) + output.emplace_back(builtin, Replxx::Color::GREEN); + + output.emplace_back("and", Replxx::Color::BRIGHTBLUE); + output.emplace_back("or", Replxx::Color::BRIGHTBLUE); + output.emplace_back("[\\-|+]?[0-9]+(\\.[0-9]+)?", Replxx::Color::YELLOW); + output.emplace_back("\".*\"", Replxx::Color::MAGENTA); + + return output; + } + std::size_t codepointLength(const std::string& str) { - std::size_t len = 0; - for (const auto c : str) - len += (c & 0xc0) != 0x80; - return len; + return std::accumulate( + str.begin(), + str.end(), + std::size_t { 0 }, + [](const std::size_t acc, const char c) { + return acc + ((c & 0xc0) != 0x80); + }); } std::size_t contextLen(const std::string& prefix) { const std::string word_break = " \t\n\r\v\f=+*&^%$#@!,./?<>;`~'\"[]{}()\\|"; - long i = static_cast(prefix.size()) - 1; std::size_t count = 0; - while (i >= 0) + for (const auto c : std::ranges::views::reverse(prefix)) { - if (word_break.find(prefix[static_cast(i)]) != std::string::npos) + if (word_break.find(c) != std::string::npos) break; - ++count; - --i; } return count; diff --git a/tests/repl/requirements.txt b/tests/repl/requirements.txt new file mode 100644 index 000000000..808fb07af --- /dev/null +++ b/tests/repl/requirements.txt @@ -0,0 +1 @@ +pexpect diff --git a/tests/repl/test.py b/tests/repl/test.py new file mode 100644 index 000000000..d166bd3d1 --- /dev/null +++ b/tests/repl/test.py @@ -0,0 +1,53 @@ +import pexpect +import sys +import os + + +def get_repl(): + executable = None + for p in ["./arkscript", "cmake-build-debug/arkscript", "build/arkscript", "build/arkscript.exe"]: + if os.path.exists(p): + executable = p + break + if executable is None: + print("Couldn't find a valid path to 'arkscript' executable") + sys.exit(1) + + child = pexpect.spawn(executable, timeout=2, encoding="utf-8") + child.logfile = sys.stdout + return child + + +def test(prompt: str, command: str, expected: str): + try: + repl.expect(prompt) + repl.send(command) + if expected is not None: + repl.expect(expected) + except pexpect.ExceptionPexpect: + print(f"Tried to match '{prompt}{command.strip()}' with '{expected}' but got {repl.after}") + raise + + +repl = get_repl() + +# repl headers +repl.expect(r"ArkScript REPL -- Version [0-9]+\.[0-9]+\.[0-9]+-[a-f0-9]{8} \[LICENSE: Mozilla Public License 2\.0\]") +repl.expect(r"Type \"quit\" to quit\. Try \"help\" for more information") + +# completion +test("main:001> ", "(le", "let") +repl.send("t a 5)\r\n") + +test("main:002> ", "(print a)\r\n", "5") + +test("main:003> ", "(im", "port") +repl.sendcontrol("c") + +test("main:003> ", "(let foo (fun (bar) (* bar 7))) (print (foo 7))\r\n", "49") + +test("main:004> ", "# just a comment\r\n", "main:005> ") + +if repl.isalive(): + repl.sendline("quit") + repl.close() diff --git a/tests/unittests/ReplSuite.cpp b/tests/unittests/ReplSuite.cpp new file mode 100644 index 000000000..71d8ef1ef --- /dev/null +++ b/tests/unittests/ReplSuite.cpp @@ -0,0 +1,38 @@ +#include + +#include + +using namespace boost; + +ut::suite<"Repl"> repl_suite = [] { + using namespace ut; + + "countOpenEnclosures"_test = [] { + expect(that % Ark::internal::countOpenEnclosures("", '(', ')') == 0); + expect(that % Ark::internal::countOpenEnclosures("(", '(', ')') == 1); + expect(that % Ark::internal::countOpenEnclosures(")", '(', ')') == -1); + expect(that % Ark::internal::countOpenEnclosures("{}", '(', ')') == 0); + expect(that % Ark::internal::countOpenEnclosures("{)(()}", '(', ')') == 0); + expect(that % Ark::internal::countOpenEnclosures("{)(()}", '{', '}') == 0); + }; + + "trimWhitespace"_test = [] { + const std::string expected = "hello world"; + std::string line = expected; + + Ark::internal::trimWhitespace(line); + expect(that % line == expected); + + line = " \thello world"; + Ark::internal::trimWhitespace(line); + expect(that % line == expected); + + line = "hello world \t"; + Ark::internal::trimWhitespace(line); + expect(that % line == expected); + + line = " \thello world \t"; + Ark::internal::trimWhitespace(line); + expect(that % line == expected); + }; +};