From 31f9c9ccbd1c9fb04236a0113286b53bb93cab58 Mon Sep 17 00:00:00 2001 From: bialger Date: Fri, 7 Nov 2025 18:47:30 +0300 Subject: [PATCH 1/3] build: add tokens library and update CMake configuration for lexer and preprocessor --- lib/CMakeLists.txt | 1 + lib/lexer/CMakeLists.txt | 19 +++---------- lib/lexer/Lexer.cpp | 6 +++- lib/lexer/Lexer.hpp | 13 +++++++-- lib/lexer/LexerError.hpp | 10 +++++-- lib/lexer/SourceCodeWrapper.cpp | 4 +++ lib/lexer/SourceCodeWrapper.hpp | 10 +++++-- lib/lexer/handlers/CharHandler.cpp | 6 +++- lib/lexer/handlers/CharHandler.hpp | 10 +++++-- lib/lexer/handlers/ColonHandler.cpp | 6 +++- lib/lexer/handlers/ColonHandler.hpp | 10 +++++-- lib/lexer/handlers/DefaultHandler.cpp | 4 +++ lib/lexer/handlers/DefaultHandler.hpp | 10 +++++-- lib/lexer/handlers/DotCompositeHandler.cpp | 4 +++ lib/lexer/handlers/DotCompositeHandler.hpp | 10 +++++-- lib/lexer/handlers/Handler.hpp | 13 ++++++--- lib/lexer/handlers/IdentifierHandler.cpp | 6 +++- lib/lexer/handlers/IdentifierHandler.hpp | 10 +++++-- lib/lexer/handlers/NewlineHandler.cpp | 6 +++- lib/lexer/handlers/NewlineHandler.hpp | 10 +++++-- lib/lexer/handlers/NumberHandler.cpp | 6 +++- lib/lexer/handlers/NumberHandler.hpp | 10 +++++-- lib/lexer/handlers/OperatorHandler.cpp | 6 +++- lib/lexer/handlers/OperatorHandler.hpp | 10 +++++-- lib/lexer/handlers/PunctHandler.cpp | 6 +++- lib/lexer/handlers/PunctHandler.hpp | 10 +++++-- lib/lexer/handlers/SlashHandler.cpp | 6 +++- lib/lexer/handlers/SlashHandler.hpp | 10 +++++-- lib/lexer/handlers/StringHandler.cpp | 6 +++- lib/lexer/handlers/StringHandler.hpp | 10 +++++-- lib/lexer/handlers/WhitespaceHandler.cpp | 4 +++ lib/lexer/handlers/WhitespaceHandler.hpp | 10 +++++-- lib/preprocessor/CMakeLists.txt | 1 + lib/preprocessor/Preprocessor.cpp | 2 +- lib/preprocessor/TokenProcessor.hpp | 4 ++- .../TokenDirectivesProcessor.hpp | 2 +- .../handlers/DefaultHandler.hpp | 2 +- .../handlers/DefineHandler.hpp | 2 +- .../handlers/DirectiveHandler.hpp | 2 +- .../handlers/ElseHandler.hpp | 2 +- .../handlers/EndifHandler.hpp | 2 +- .../handlers/IfdefHandler.hpp | 2 +- .../handlers/IfndefHandler.hpp | 2 +- .../handlers/UndefHandler.hpp | 2 +- .../import_processor/TokenImportProcessor.cpp | 2 +- .../import_processor/TokenImportProcessor.hpp | 2 +- lib/tokens/CMakeLists.txt | 28 +++++++++++++++++++ lib/{lexer => }/tokens/CommentToken.cpp | 4 +++ lib/{lexer => }/tokens/CommentToken.hpp | 10 +++++-- lib/{lexer => }/tokens/EofToken.cpp | 4 +++ lib/{lexer => }/tokens/EofToken.hpp | 10 +++++-- lib/{lexer => }/tokens/IdentToken.cpp | 4 +++ lib/{lexer => }/tokens/IdentToken.hpp | 10 +++++-- lib/{lexer => }/tokens/KeywordToken.cpp | 4 +++ lib/{lexer => }/tokens/KeywordToken.hpp | 10 +++++-- lib/{lexer => }/tokens/LiteralToken.cpp | 4 +++ lib/{lexer => }/tokens/LiteralToken.hpp | 12 +++++--- lib/{lexer => }/tokens/NewlineToken.cpp | 4 +++ lib/{lexer => }/tokens/NewlineToken.hpp | 10 +++++-- lib/{lexer => }/tokens/OperatorToken.cpp | 4 +++ lib/{lexer => }/tokens/OperatorToken.hpp | 10 +++++-- lib/{lexer => }/tokens/PunctToken.cpp | 4 +++ lib/{lexer => }/tokens/PunctToken.hpp | 10 +++++-- lib/{lexer => }/tokens/Token.hpp | 10 +++++-- lib/{lexer => }/tokens/TokenFactory.cpp | 16 +++++++---- lib/{lexer => }/tokens/TokenFactory.hpp | 10 +++---- lib/{lexer => }/tokens/TokenPosition.cpp | 4 +++ lib/{lexer => }/tokens/TokenPosition.hpp | 10 +++++-- lib/{lexer => }/tokens/TokenVisitor.hpp | 10 +++++-- lib/{lexer => tokens}/values/BoolValue.cpp | 4 +++ lib/{lexer => tokens}/values/BoolValue.hpp | 10 +++++-- lib/{lexer => tokens}/values/CharValue.cpp | 4 +++ lib/{lexer => tokens}/values/CharValue.hpp | 10 +++++-- lib/{lexer => tokens}/values/FloatValue.cpp | 4 +++ lib/{lexer => tokens}/values/FloatValue.hpp | 10 +++++-- lib/{lexer => tokens}/values/IntValue.cpp | 4 +++ lib/{lexer => tokens}/values/IntValue.hpp | 10 +++++-- lib/{lexer => tokens}/values/StringValue.cpp | 4 +++ lib/{lexer => tokens}/values/StringValue.hpp | 10 +++++-- lib/{lexer => tokens}/values/Value.hpp | 10 +++++-- tests/lexer_big_programs_tests.cpp | 2 ++ tests/lexer_tests.cpp | 2 ++ tests/test_suites/LexerUnitTestSuite.cpp | 2 +- tests/test_suites/LexerUnitTestSuite.hpp | 6 ++-- .../test_suites/PreprocessorUnitTestSuite.cpp | 2 +- .../test_suites/PreprocessorUnitTestSuite.hpp | 2 +- .../ProjectIntegrationTestSuite.hpp | 1 + 87 files changed, 435 insertions(+), 155 deletions(-) create mode 100644 lib/tokens/CMakeLists.txt rename lib/{lexer => }/tokens/CommentToken.cpp (95%) rename lib/{lexer => }/tokens/CommentToken.hpp (81%) rename lib/{lexer => }/tokens/EofToken.cpp (94%) rename lib/{lexer => }/tokens/EofToken.hpp (81%) rename lib/{lexer => }/tokens/IdentToken.cpp (95%) rename lib/{lexer => }/tokens/IdentToken.hpp (83%) rename lib/{lexer => }/tokens/KeywordToken.cpp (95%) rename lib/{lexer => }/tokens/KeywordToken.hpp (81%) rename lib/{lexer => }/tokens/LiteralToken.cpp (96%) rename lib/{lexer => }/tokens/LiteralToken.hpp (82%) rename lib/{lexer => }/tokens/NewlineToken.cpp (95%) rename lib/{lexer => }/tokens/NewlineToken.hpp (81%) rename lib/{lexer => }/tokens/OperatorToken.cpp (95%) rename lib/{lexer => }/tokens/OperatorToken.hpp (81%) rename lib/{lexer => }/tokens/PunctToken.cpp (95%) rename lib/{lexer => }/tokens/PunctToken.hpp (83%) rename lib/{lexer => }/tokens/Token.hpp (84%) rename lib/{lexer => }/tokens/TokenFactory.cpp (90%) rename lib/{lexer => }/tokens/TokenFactory.hpp (81%) rename lib/{lexer => }/tokens/TokenPosition.cpp (87%) rename lib/{lexer => }/tokens/TokenPosition.hpp (63%) rename lib/{lexer => }/tokens/TokenVisitor.hpp (82%) rename lib/{lexer => tokens}/values/BoolValue.cpp (88%) rename lib/{lexer => tokens}/values/BoolValue.hpp (71%) rename lib/{lexer => tokens}/values/CharValue.cpp (88%) rename lib/{lexer => tokens}/values/CharValue.hpp (71%) rename lib/{lexer => tokens}/values/FloatValue.cpp (90%) rename lib/{lexer => tokens}/values/FloatValue.hpp (71%) rename lib/{lexer => tokens}/values/IntValue.cpp (88%) rename lib/{lexer => tokens}/values/IntValue.hpp (73%) rename lib/{lexer => tokens}/values/StringValue.cpp (89%) rename lib/{lexer => tokens}/values/StringValue.hpp (71%) rename lib/{lexer => tokens}/values/Value.hpp (73%) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index bd0fc3a..e6238d1 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -1,3 +1,4 @@ +add_subdirectory(tokens) add_subdirectory(lexer) add_subdirectory(preprocessor) add_subdirectory(compiler_ui) diff --git a/lib/lexer/CMakeLists.txt b/lib/lexer/CMakeLists.txt index 4a04cbe..648365d 100644 --- a/lib/lexer/CMakeLists.txt +++ b/lib/lexer/CMakeLists.txt @@ -13,21 +13,6 @@ add_library(lexer STATIC handlers/SlashHandler.cpp handlers/StringHandler.cpp handlers/WhitespaceHandler.cpp - tokens/CommentToken.cpp - tokens/EofToken.cpp - tokens/IdentToken.cpp - tokens/KeywordToken.cpp - tokens/LiteralToken.cpp - tokens/NewlineToken.cpp - tokens/OperatorToken.cpp - tokens/PunctToken.cpp - values/BoolValue.cpp - values/CharValue.cpp - values/FloatValue.cpp - values/IntValue.cpp - values/StringValue.cpp - tokens/TokenFactory.cpp - tokens/TokenPosition.cpp ) target_include_directories(lexer @@ -39,3 +24,7 @@ target_include_directories(lexer PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ) + +target_link_libraries(lexer PUBLIC + tokens +) diff --git a/lib/lexer/Lexer.cpp b/lib/lexer/Lexer.cpp index 46a84c7..ffd59fc 100644 --- a/lib/lexer/Lexer.cpp +++ b/lib/lexer/Lexer.cpp @@ -16,7 +16,9 @@ #include "handlers/StringHandler.hpp" #include "handlers/WhitespaceHandler.hpp" -#include "tokens/TokenFactory.hpp" +#include "lib/tokens/TokenFactory.hpp" + +namespace ovum::compiler::lexer { namespace { constexpr const char* kOperatorChars = "+-*%<>=!&|^~?"; @@ -105,3 +107,5 @@ std::array, kDefaultTokenReserve> Lexer::MakeDefaultHan std::unique_ptr Lexer::MakeDefaultHandler() { return std::make_unique(); } + +} // namespace ovum::compiler::lexer diff --git a/lib/lexer/Lexer.hpp b/lib/lexer/Lexer.hpp index 39bfdd0..a7f471c 100644 --- a/lib/lexer/Lexer.hpp +++ b/lib/lexer/Lexer.hpp @@ -1,5 +1,5 @@ -#ifndef LEXER_HPP_ -#define LEXER_HPP_ +#ifndef LEXER_LEXER_HPP_ +#define LEXER_LEXER_HPP_ #include #include @@ -9,6 +9,11 @@ #include "SourceCodeWrapper.hpp" #include "handlers/Handler.hpp" +#include "lib/tokens/Token.hpp" + +namespace ovum::compiler::lexer { + +using TokenPtr = ovum::TokenPtr; constexpr std::size_t kDefaultTokenReserve = 256; @@ -32,4 +37,6 @@ class Lexer { std::unique_ptr default_handler_; }; -#endif // LEXER_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_LEXER_HPP_ diff --git a/lib/lexer/LexerError.hpp b/lib/lexer/LexerError.hpp index 0c55e8a..2c859cf 100644 --- a/lib/lexer/LexerError.hpp +++ b/lib/lexer/LexerError.hpp @@ -1,11 +1,15 @@ -#ifndef LEXERERROR_HPP_ -#define LEXERERROR_HPP_ +#ifndef LEXER_LEXERERROR_HPP_ +#define LEXER_LEXERERROR_HPP_ #include +namespace ovum::compiler::lexer { + class LexerError : public std::runtime_error { public: using std::runtime_error::runtime_error; }; -#endif // LEXERERROR_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_LEXERERROR_HPP_ diff --git a/lib/lexer/SourceCodeWrapper.cpp b/lib/lexer/SourceCodeWrapper.cpp index a5033bc..ec76a9d 100644 --- a/lib/lexer/SourceCodeWrapper.cpp +++ b/lib/lexer/SourceCodeWrapper.cpp @@ -2,6 +2,8 @@ #include +namespace ovum::compiler::lexer { + const std::unordered_set SourceCodeWrapper::kKeywords = { "fun", "class", "interface", "var", "override", "pure", "if", "else", "continue", "break", "for", "while", "return", "unsafe", "val", "static", "public", "private", @@ -121,3 +123,5 @@ bool SourceCodeWrapper::IsKeyword(std::string_view s) { bool SourceCodeWrapper::IsMultiOp(std::string_view s) { return kMultiOps.contains(s); } + +} // namespace ovum::compiler::lexer diff --git a/lib/lexer/SourceCodeWrapper.hpp b/lib/lexer/SourceCodeWrapper.hpp index c6094c0..207c60d 100644 --- a/lib/lexer/SourceCodeWrapper.hpp +++ b/lib/lexer/SourceCodeWrapper.hpp @@ -1,5 +1,5 @@ -#ifndef SOURCECODEWRAPPER_HPP_ -#define SOURCECODEWRAPPER_HPP_ +#ifndef LEXER_SOURCECODEWRAPPER_HPP_ +#define LEXER_SOURCECODEWRAPPER_HPP_ #include #include @@ -7,6 +7,8 @@ #include #include +namespace ovum::compiler::lexer { + class SourceCodeWrapper { public: SourceCodeWrapper(std::string_view src, bool keep_comments = false); @@ -53,4 +55,6 @@ class SourceCodeWrapper { static const std::unordered_set kMultiOps; }; -#endif // SOURCECODEWRAPPER_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_SOURCECODEWRAPPER_HPP_ diff --git a/lib/lexer/handlers/CharHandler.cpp b/lib/lexer/handlers/CharHandler.cpp index 7f7a54f..47fd764 100644 --- a/lib/lexer/handlers/CharHandler.cpp +++ b/lib/lexer/handlers/CharHandler.cpp @@ -1,7 +1,9 @@ #include "CharHandler.hpp" #include "lib/lexer/LexerError.hpp" -#include "tokens/TokenFactory.hpp" +#include "lib/tokens/TokenFactory.hpp" + +namespace ovum::compiler::lexer { OptToken CharHandler::Scan(SourceCodeWrapper& wrapper) { std::string raw; @@ -56,3 +58,5 @@ OptToken CharHandler::Scan(SourceCodeWrapper& wrapper) { return std::make_optional( TokenFactory::MakeCharLiteral(std::move(raw), val, wrapper.GetLine(), wrapper.GetTokenCol())); } + +} // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/CharHandler.hpp b/lib/lexer/handlers/CharHandler.hpp index 6d07ebf..fc8baa9 100644 --- a/lib/lexer/handlers/CharHandler.hpp +++ b/lib/lexer/handlers/CharHandler.hpp @@ -1,11 +1,15 @@ -#ifndef CHARHANDLER_HPP_ -#define CHARHANDLER_HPP_ +#ifndef LEXER_CHARHANDLER_HPP_ +#define LEXER_CHARHANDLER_HPP_ #include "Handler.hpp" +namespace ovum::compiler::lexer { + class CharHandler : public Handler { public: OptToken Scan(SourceCodeWrapper& wrapper) override; }; -#endif // CHARHANDLER_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_CHARHANDLER_HPP_ diff --git a/lib/lexer/handlers/ColonHandler.cpp b/lib/lexer/handlers/ColonHandler.cpp index 6807b91..ef57089 100644 --- a/lib/lexer/handlers/ColonHandler.cpp +++ b/lib/lexer/handlers/ColonHandler.cpp @@ -1,5 +1,7 @@ #include "ColonHandler.hpp" -#include "lib/lexer/tokens/TokenFactory.hpp" +#include "lib/tokens/TokenFactory.hpp" + +namespace ovum::compiler::lexer { OptToken ColonHandler::Scan(SourceCodeWrapper& wrapper) { std::string op; @@ -18,3 +20,5 @@ OptToken ColonHandler::Scan(SourceCodeWrapper& wrapper) { return std::make_optional(TokenFactory::MakePunct(':', wrapper.GetLine(), wrapper.GetTokenCol())); } + +} // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/ColonHandler.hpp b/lib/lexer/handlers/ColonHandler.hpp index 9184ff5..fbdbb74 100644 --- a/lib/lexer/handlers/ColonHandler.hpp +++ b/lib/lexer/handlers/ColonHandler.hpp @@ -1,10 +1,14 @@ -#ifndef COLONHANDLER_HPP_ -#define COLONHANDLER_HPP_ +#ifndef LEXER_COLONHANDLER_HPP_ +#define LEXER_COLONHANDLER_HPP_ #include "Handler.hpp" +namespace ovum::compiler::lexer { + class ColonHandler : public Handler { public: OptToken Scan(SourceCodeWrapper& wrapper) override; }; -#endif // COLONHANDLER_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_COLONHANDLER_HPP_ diff --git a/lib/lexer/handlers/DefaultHandler.cpp b/lib/lexer/handlers/DefaultHandler.cpp index e87bc8c..dbfc99d 100644 --- a/lib/lexer/handlers/DefaultHandler.cpp +++ b/lib/lexer/handlers/DefaultHandler.cpp @@ -2,7 +2,11 @@ #include "lib/lexer/LexerError.hpp" +namespace ovum::compiler::lexer { + OptToken DefaultHandler::Scan(SourceCodeWrapper& wrapper) { char c = wrapper.CurrentChar(); throw LexerError(std::string("Unexpected character: ") + c); } + +} // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/DefaultHandler.hpp b/lib/lexer/handlers/DefaultHandler.hpp index 9374a84..85dc411 100644 --- a/lib/lexer/handlers/DefaultHandler.hpp +++ b/lib/lexer/handlers/DefaultHandler.hpp @@ -1,11 +1,15 @@ -#ifndef DEFAULTHANDLER_HPP_ -#define DEFAULTHANDLER_HPP_ +#ifndef LEXER_DEFAULTHANDLER_HPP_ +#define LEXER_DEFAULTHANDLER_HPP_ #include "Handler.hpp" +namespace ovum::compiler::lexer { + class DefaultHandler : public Handler { public: OptToken Scan(SourceCodeWrapper& wrapper) override; }; -#endif // DEFAULTHANDLER_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_DEFAULTHANDLER_HPP_ diff --git a/lib/lexer/handlers/DotCompositeHandler.cpp b/lib/lexer/handlers/DotCompositeHandler.cpp index df0f202..7d6a6b1 100644 --- a/lib/lexer/handlers/DotCompositeHandler.cpp +++ b/lib/lexer/handlers/DotCompositeHandler.cpp @@ -1,5 +1,7 @@ #include "DotCompositeHandler.hpp" +namespace ovum::compiler::lexer { + DotCompositeHandler::DotCompositeHandler() : num_(std::make_unique()), op_(std::make_unique()), punct_(std::make_unique()) { @@ -20,3 +22,5 @@ OptToken DotCompositeHandler::Scan(SourceCodeWrapper& w) { return std::nullopt; } + +} // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/DotCompositeHandler.hpp b/lib/lexer/handlers/DotCompositeHandler.hpp index 4eb8a1c..ea3297a 100644 --- a/lib/lexer/handlers/DotCompositeHandler.hpp +++ b/lib/lexer/handlers/DotCompositeHandler.hpp @@ -1,11 +1,13 @@ -#ifndef DOTCOMPOSITEHANDLER_HPP_ -#define DOTCOMPOSITEHANDLER_HPP_ +#ifndef LEXER_DOTCOMPOSITEHANDLER_HPP_ +#define LEXER_DOTCOMPOSITEHANDLER_HPP_ #include "Handler.hpp" #include "NumberHandler.hpp" #include "OperatorHandler.hpp" #include "PunctHandler.hpp" +namespace ovum::compiler::lexer { + class DotCompositeHandler : public Handler { public: DotCompositeHandler(); @@ -18,4 +20,6 @@ class DotCompositeHandler : public Handler { std::unique_ptr punct_; }; -#endif // DOTCOMPOSITEHANDLER_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_DOTCOMPOSITEHANDLER_HPP_ diff --git a/lib/lexer/handlers/Handler.hpp b/lib/lexer/handlers/Handler.hpp index bd960c1..b6f4db4 100644 --- a/lib/lexer/handlers/Handler.hpp +++ b/lib/lexer/handlers/Handler.hpp @@ -1,11 +1,14 @@ -#ifndef HANDLER_HPP_ -#define HANDLER_HPP_ +#ifndef LEXER_HANDLER_HPP_ +#define LEXER_HANDLER_HPP_ #include #include "lib/lexer/SourceCodeWrapper.hpp" -#include "lib/lexer/tokens/Token.hpp" +#include "lib/tokens/Token.hpp" +namespace ovum::compiler::lexer { + +using TokenPtr = ovum::TokenPtr; using OptToken = std::optional; class Handler { // NOLINT(cppcoreguidelines-special-member-functions) @@ -15,4 +18,6 @@ class Handler { // NOLINT(cppcoreguidelines-special-member-functions) virtual OptToken Scan(SourceCodeWrapper& wrapper) = 0; }; -#endif // HANDLER_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_HANDLER_HPP_ diff --git a/lib/lexer/handlers/IdentifierHandler.cpp b/lib/lexer/handlers/IdentifierHandler.cpp index bebe045..c1ee429 100644 --- a/lib/lexer/handlers/IdentifierHandler.cpp +++ b/lib/lexer/handlers/IdentifierHandler.cpp @@ -4,7 +4,9 @@ #include #include "LexerError.hpp" -#include "lib/lexer/tokens/TokenFactory.hpp" +#include "lib/tokens/TokenFactory.hpp" + +namespace ovum::compiler::lexer { OptToken IdentifierHandler::Scan(SourceCodeWrapper& wrapper) { std::string s; @@ -36,3 +38,5 @@ OptToken IdentifierHandler::Scan(SourceCodeWrapper& wrapper) { return std::make_optional(TokenFactory::MakeIdent(std::move(s), wrapper.GetLine(), wrapper.GetTokenCol())); } + +} // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/IdentifierHandler.hpp b/lib/lexer/handlers/IdentifierHandler.hpp index fbabc98..e00a7f0 100644 --- a/lib/lexer/handlers/IdentifierHandler.hpp +++ b/lib/lexer/handlers/IdentifierHandler.hpp @@ -1,11 +1,15 @@ -#ifndef IDENTIFIERHANDLER_HPP_ -#define IDENTIFIERHANDLER_HPP_ +#ifndef LEXER_IDENTIFIERHANDLER_HPP_ +#define LEXER_IDENTIFIERHANDLER_HPP_ #include "Handler.hpp" +namespace ovum::compiler::lexer { + class IdentifierHandler : public Handler { public: OptToken Scan(SourceCodeWrapper& wrapper) override; }; -#endif // IDENTIFIERHANDLER_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_IDENTIFIERHANDLER_HPP_ diff --git a/lib/lexer/handlers/NewlineHandler.cpp b/lib/lexer/handlers/NewlineHandler.cpp index de5b775..e6ae91a 100644 --- a/lib/lexer/handlers/NewlineHandler.cpp +++ b/lib/lexer/handlers/NewlineHandler.cpp @@ -1,7 +1,11 @@ #include "NewlineHandler.hpp" -#include "lib/lexer/tokens/TokenFactory.hpp" +#include "lib/tokens/TokenFactory.hpp" + +namespace ovum::compiler::lexer { OptToken NewlineHandler::Scan(SourceCodeWrapper& wrapper) { return std::make_optional(TokenFactory::MakeNewline(wrapper.GetLine() - 1, wrapper.GetTokenCol())); } + +} // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/NewlineHandler.hpp b/lib/lexer/handlers/NewlineHandler.hpp index 42cd181..025df47 100644 --- a/lib/lexer/handlers/NewlineHandler.hpp +++ b/lib/lexer/handlers/NewlineHandler.hpp @@ -1,11 +1,15 @@ -#ifndef NEWLINEHANDLER_HPP_ -#define NEWLINEHANDLER_HPP_ +#ifndef LEXER_NEWLINEHANDLER_HPP_ +#define LEXER_NEWLINEHANDLER_HPP_ #include "Handler.hpp" +namespace ovum::compiler::lexer { + class NewlineHandler : public Handler { public: OptToken Scan(SourceCodeWrapper& wrapper) override; }; -#endif // NEWLINEHANDLER_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_NEWLINEHANDLER_HPP_ diff --git a/lib/lexer/handlers/NumberHandler.cpp b/lib/lexer/handlers/NumberHandler.cpp index 9c30220..fe2977c 100644 --- a/lib/lexer/handlers/NumberHandler.cpp +++ b/lib/lexer/handlers/NumberHandler.cpp @@ -4,7 +4,9 @@ #include #include "lib/lexer/LexerError.hpp" -#include "tokens/TokenFactory.hpp" +#include "lib/tokens/TokenFactory.hpp" + +namespace ovum::compiler::lexer { namespace { inline constexpr int kHexAlphaOffset = 10; @@ -189,3 +191,5 @@ OptToken NumberHandler::Scan(SourceCodeWrapper& wrapper) { return TokenFactory::MakeIntLiteral(std::move(raw), vi, wrapper.GetLine(), wrapper.GetTokenCol()); } + +} // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/NumberHandler.hpp b/lib/lexer/handlers/NumberHandler.hpp index 2dfcad9..ef33f9a 100644 --- a/lib/lexer/handlers/NumberHandler.hpp +++ b/lib/lexer/handlers/NumberHandler.hpp @@ -1,11 +1,15 @@ -#ifndef NUMBERHANDLER_HPP_ -#define NUMBERHANDLER_HPP_ +#ifndef LEXER_NUMBERHANDLER_HPP_ +#define LEXER_NUMBERHANDLER_HPP_ #include "Handler.hpp" +namespace ovum::compiler::lexer { + class NumberHandler : public Handler { public: OptToken Scan(SourceCodeWrapper& wrapper) override; }; -#endif // NUMBERHANDLER_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_NUMBERHANDLER_HPP_ diff --git a/lib/lexer/handlers/OperatorHandler.cpp b/lib/lexer/handlers/OperatorHandler.cpp index 4c57f4b..ae0f67b 100644 --- a/lib/lexer/handlers/OperatorHandler.cpp +++ b/lib/lexer/handlers/OperatorHandler.cpp @@ -1,6 +1,8 @@ #include "OperatorHandler.hpp" -#include "lib/lexer/tokens/TokenFactory.hpp" +#include "lib/tokens/TokenFactory.hpp" + +namespace ovum::compiler::lexer { OptToken OperatorHandler::Scan(SourceCodeWrapper& wrapper) { std::string op; @@ -18,3 +20,5 @@ OptToken OperatorHandler::Scan(SourceCodeWrapper& wrapper) { return std::make_optional(TokenFactory::MakeOperator(std::move(op), wrapper.GetLine(), wrapper.GetTokenCol())); } + +} // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/OperatorHandler.hpp b/lib/lexer/handlers/OperatorHandler.hpp index 1185a8f..e8168d7 100644 --- a/lib/lexer/handlers/OperatorHandler.hpp +++ b/lib/lexer/handlers/OperatorHandler.hpp @@ -1,11 +1,15 @@ -#ifndef OPERATORHANDLER_HPP_ -#define OPERATORHANDLER_HPP_ +#ifndef LEXER_OPERATORHANDLER_HPP_ +#define LEXER_OPERATORHANDLER_HPP_ #include "Handler.hpp" +namespace ovum::compiler::lexer { + class OperatorHandler : public Handler { public: OptToken Scan(SourceCodeWrapper& wrapper) override; }; -#endif // OPERATORHANDLER_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_OPERATORHANDLER_HPP_ diff --git a/lib/lexer/handlers/PunctHandler.cpp b/lib/lexer/handlers/PunctHandler.cpp index 28bc6f8..3a3b3ec 100644 --- a/lib/lexer/handlers/PunctHandler.cpp +++ b/lib/lexer/handlers/PunctHandler.cpp @@ -1,8 +1,12 @@ #include "PunctHandler.hpp" -#include "lib/lexer/tokens/TokenFactory.hpp" +#include "lib/tokens/TokenFactory.hpp" + +namespace ovum::compiler::lexer { OptToken PunctHandler::Scan(SourceCodeWrapper& wrapper) { char c = wrapper.CurrentChar(); return std::make_optional(TokenFactory::MakePunct(c, wrapper.GetLine(), wrapper.GetTokenCol())); } + +} // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/PunctHandler.hpp b/lib/lexer/handlers/PunctHandler.hpp index d69e5a8..bbc6296 100644 --- a/lib/lexer/handlers/PunctHandler.hpp +++ b/lib/lexer/handlers/PunctHandler.hpp @@ -1,11 +1,15 @@ -#ifndef PUNCTHANDLER_HPP_ -#define PUNCTHANDLER_HPP_ +#ifndef LEXER_PUNCTHANDLER_HPP_ +#define LEXER_PUNCTHANDLER_HPP_ #include "Handler.hpp" +namespace ovum::compiler::lexer { + class PunctHandler : public Handler { public: OptToken Scan(SourceCodeWrapper& wrapper) override; }; -#endif // PUNCTHANDLER_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_PUNCTHANDLER_HPP_ diff --git a/lib/lexer/handlers/SlashHandler.cpp b/lib/lexer/handlers/SlashHandler.cpp index 035645e..76b2796 100644 --- a/lib/lexer/handlers/SlashHandler.cpp +++ b/lib/lexer/handlers/SlashHandler.cpp @@ -1,7 +1,9 @@ #include "SlashHandler.hpp" #include "lib/lexer/LexerError.hpp" -#include "tokens/TokenFactory.hpp" +#include "lib/tokens/TokenFactory.hpp" + +namespace ovum::compiler::lexer { OptToken SlashHandler::Scan(SourceCodeWrapper& wrapper) { if (wrapper.Peek() == '/') { @@ -50,3 +52,5 @@ OptToken SlashHandler::Scan(SourceCodeWrapper& wrapper) { return std::make_optional(TokenFactory::MakeOperator(std::string(1, '/'), wrapper.GetLine(), wrapper.GetTokenCol())); } + +} // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/SlashHandler.hpp b/lib/lexer/handlers/SlashHandler.hpp index 369c5a3..5329841 100644 --- a/lib/lexer/handlers/SlashHandler.hpp +++ b/lib/lexer/handlers/SlashHandler.hpp @@ -1,11 +1,15 @@ -#ifndef SLASHHANDLER_HPP_ -#define SLASHHANDLER_HPP_ +#ifndef LEXER_SLASHHANDLER_HPP_ +#define LEXER_SLASHHANDLER_HPP_ #include "Handler.hpp" +namespace ovum::compiler::lexer { + class SlashHandler : public Handler { public: OptToken Scan(SourceCodeWrapper& wrapper) override; }; -#endif // SLASHHANDLER_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_SLASHHANDLER_HPP_ diff --git a/lib/lexer/handlers/StringHandler.cpp b/lib/lexer/handlers/StringHandler.cpp index c1eb07c..ead4537 100644 --- a/lib/lexer/handlers/StringHandler.cpp +++ b/lib/lexer/handlers/StringHandler.cpp @@ -3,7 +3,9 @@ #include #include "lib/lexer/LexerError.hpp" -#include "tokens/TokenFactory.hpp" +#include "lib/tokens/TokenFactory.hpp" + +namespace ovum::compiler::lexer { OptToken StringHandler::Scan(SourceCodeWrapper& wrapper) { std::string raw; @@ -60,3 +62,5 @@ OptToken StringHandler::Scan(SourceCodeWrapper& wrapper) { throw LexerError("Unterminated string literal (EOF reached)"); } + +} // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/StringHandler.hpp b/lib/lexer/handlers/StringHandler.hpp index a735134..78720f9 100644 --- a/lib/lexer/handlers/StringHandler.hpp +++ b/lib/lexer/handlers/StringHandler.hpp @@ -1,11 +1,15 @@ -#ifndef STRINGHANDLER_H -#define STRINGHANDLER_H +#ifndef LEXER_STRINGHANDLER_HPP_ +#define LEXER_STRINGHANDLER_HPP_ #include "Handler.hpp" +namespace ovum::compiler::lexer { + class StringHandler : public Handler { public: OptToken Scan(SourceCodeWrapper& wrapper) override; }; -#endif // STRINGHANDLER_H +} // namespace ovum::compiler::lexer + +#endif // LEXER_STRINGHANDLER_HPP_ diff --git a/lib/lexer/handlers/WhitespaceHandler.cpp b/lib/lexer/handlers/WhitespaceHandler.cpp index d2f1d1d..694eb74 100644 --- a/lib/lexer/handlers/WhitespaceHandler.cpp +++ b/lib/lexer/handlers/WhitespaceHandler.cpp @@ -1,5 +1,7 @@ #include "WhitespaceHandler.hpp" +namespace ovum::compiler::lexer { + OptToken WhitespaceHandler::Scan(SourceCodeWrapper& wrapper) { while (!wrapper.IsAtEnd()) { if (char p = wrapper.Peek(); p != ' ' && p != '\t' && p != '\r') { @@ -11,3 +13,5 @@ OptToken WhitespaceHandler::Scan(SourceCodeWrapper& wrapper) { return std::nullopt; } + +} // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/WhitespaceHandler.hpp b/lib/lexer/handlers/WhitespaceHandler.hpp index 3bab9db..e3bf5d4 100644 --- a/lib/lexer/handlers/WhitespaceHandler.hpp +++ b/lib/lexer/handlers/WhitespaceHandler.hpp @@ -1,11 +1,15 @@ -#ifndef WHITESPACEHANDLER_HPP_ -#define WHITESPACEHANDLER_HPP_ +#ifndef LEXER_WHITESPACEHANDLER_HPP_ +#define LEXER_WHITESPACEHANDLER_HPP_ #include "Handler.hpp" +namespace ovum::compiler::lexer { + class WhitespaceHandler : public Handler { public: OptToken Scan(SourceCodeWrapper& wrapper) override; }; -#endif // WHITESPACEHANDLER_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_WHITESPACEHANDLER_HPP_ diff --git a/lib/preprocessor/CMakeLists.txt b/lib/preprocessor/CMakeLists.txt index 3631006..d0c84ec 100644 --- a/lib/preprocessor/CMakeLists.txt +++ b/lib/preprocessor/CMakeLists.txt @@ -26,6 +26,7 @@ target_include_directories(preprocessor target_link_libraries(preprocessor PUBLIC lexer + tokens ) target_include_directories(preprocessor PUBLIC ${PROJECT_SOURCE_DIR}) diff --git a/lib/preprocessor/Preprocessor.cpp b/lib/preprocessor/Preprocessor.cpp index 1d58687..adecb8d 100644 --- a/lib/preprocessor/Preprocessor.cpp +++ b/lib/preprocessor/Preprocessor.cpp @@ -32,7 +32,7 @@ std::expected, PreprocessorError> Preprocessor::Process() return std::unexpected(FileReadError(file.string(), "Read error")); } - Lexer lexer(content, false); + lexer::Lexer lexer(content, false); std::vector tokens; try { diff --git a/lib/preprocessor/TokenProcessor.hpp b/lib/preprocessor/TokenProcessor.hpp index 2224f23..99dc07f 100644 --- a/lib/preprocessor/TokenProcessor.hpp +++ b/lib/preprocessor/TokenProcessor.hpp @@ -5,10 +5,12 @@ #include #include "PreprocessorError.hpp" -#include "lib/lexer/tokens/Token.hpp" +#include "lib/tokens/Token.hpp" namespace ovum::compiler::preprocessor { +using TokenPtr = ovum::TokenPtr; + class TokenProcessor { // NOLINT(cppcoreguidelines-special-member-functions) public: virtual ~TokenProcessor() = default; diff --git a/lib/preprocessor/directives_processor/TokenDirectivesProcessor.hpp b/lib/preprocessor/directives_processor/TokenDirectivesProcessor.hpp index 25f3e23..0317934 100644 --- a/lib/preprocessor/directives_processor/TokenDirectivesProcessor.hpp +++ b/lib/preprocessor/directives_processor/TokenDirectivesProcessor.hpp @@ -7,10 +7,10 @@ #include #include -#include "lib/lexer/tokens/Token.hpp" #include "lib/preprocessor/PreprocessorError.hpp" #include "lib/preprocessor/TokenProcessor.hpp" #include "lib/preprocessor/directives_processor/handlers/DirectiveHandler.hpp" +#include "lib/tokens/Token.hpp" namespace ovum::compiler::preprocessor { diff --git a/lib/preprocessor/directives_processor/handlers/DefaultHandler.hpp b/lib/preprocessor/directives_processor/handlers/DefaultHandler.hpp index a3dc344..879d15e 100644 --- a/lib/preprocessor/directives_processor/handlers/DefaultHandler.hpp +++ b/lib/preprocessor/directives_processor/handlers/DefaultHandler.hpp @@ -2,7 +2,7 @@ #define PREPROCESSOR_DEFAULT_HANDLER_HPP_ #include "DirectiveHandler.hpp" -#include "lib/lexer/tokens/Token.hpp" +#include "lib/tokens/Token.hpp" namespace ovum::compiler::preprocessor { diff --git a/lib/preprocessor/directives_processor/handlers/DefineHandler.hpp b/lib/preprocessor/directives_processor/handlers/DefineHandler.hpp index 3ac0957..56c03a6 100644 --- a/lib/preprocessor/directives_processor/handlers/DefineHandler.hpp +++ b/lib/preprocessor/directives_processor/handlers/DefineHandler.hpp @@ -2,7 +2,7 @@ #define PREPROCESSOR_DEFINE_HANDLER_HPP_ #include "DirectiveHandler.hpp" -#include "lib/lexer/tokens/Token.hpp" +#include "lib/tokens/Token.hpp" namespace ovum::compiler::preprocessor { diff --git a/lib/preprocessor/directives_processor/handlers/DirectiveHandler.hpp b/lib/preprocessor/directives_processor/handlers/DirectiveHandler.hpp index e182e86..6e50552 100644 --- a/lib/preprocessor/directives_processor/handlers/DirectiveHandler.hpp +++ b/lib/preprocessor/directives_processor/handlers/DirectiveHandler.hpp @@ -7,8 +7,8 @@ #include #include -#include "lib/lexer/tokens/Token.hpp" #include "lib/preprocessor/PreprocessorError.hpp" +#include "lib/tokens/Token.hpp" namespace ovum::compiler::preprocessor { diff --git a/lib/preprocessor/directives_processor/handlers/ElseHandler.hpp b/lib/preprocessor/directives_processor/handlers/ElseHandler.hpp index c4d3af9..79cece4 100644 --- a/lib/preprocessor/directives_processor/handlers/ElseHandler.hpp +++ b/lib/preprocessor/directives_processor/handlers/ElseHandler.hpp @@ -2,7 +2,7 @@ #define PREPROCESSOR_ELSE_HANDLER_HPP_ #include "DirectiveHandler.hpp" -#include "lib/lexer/tokens/Token.hpp" +#include "lib/tokens/Token.hpp" namespace ovum::compiler::preprocessor { diff --git a/lib/preprocessor/directives_processor/handlers/EndifHandler.hpp b/lib/preprocessor/directives_processor/handlers/EndifHandler.hpp index 88d1e5f..bce6fae 100644 --- a/lib/preprocessor/directives_processor/handlers/EndifHandler.hpp +++ b/lib/preprocessor/directives_processor/handlers/EndifHandler.hpp @@ -2,7 +2,7 @@ #define PREPROCESSOR_ENDIF_HANDLER_HPP_ #include "DirectiveHandler.hpp" -#include "lib/lexer/tokens/Token.hpp" +#include "lib/tokens/Token.hpp" namespace ovum::compiler::preprocessor { diff --git a/lib/preprocessor/directives_processor/handlers/IfdefHandler.hpp b/lib/preprocessor/directives_processor/handlers/IfdefHandler.hpp index eeebe2f..7e9baee 100644 --- a/lib/preprocessor/directives_processor/handlers/IfdefHandler.hpp +++ b/lib/preprocessor/directives_processor/handlers/IfdefHandler.hpp @@ -2,7 +2,7 @@ #define PREPROCESSOR_IFDEF_HANDLER_HPP_ #include "DirectiveHandler.hpp" -#include "lib/lexer/tokens/Token.hpp" +#include "lib/tokens/Token.hpp" namespace ovum::compiler::preprocessor { diff --git a/lib/preprocessor/directives_processor/handlers/IfndefHandler.hpp b/lib/preprocessor/directives_processor/handlers/IfndefHandler.hpp index 81468f9..cf7a216 100644 --- a/lib/preprocessor/directives_processor/handlers/IfndefHandler.hpp +++ b/lib/preprocessor/directives_processor/handlers/IfndefHandler.hpp @@ -2,7 +2,7 @@ #define PREPROCESSOR_IFNDEF_HANDLER_HPP_ #include "DirectiveHandler.hpp" -#include "lib/lexer/tokens/Token.hpp" +#include "lib/tokens/Token.hpp" namespace ovum::compiler::preprocessor { diff --git a/lib/preprocessor/directives_processor/handlers/UndefHandler.hpp b/lib/preprocessor/directives_processor/handlers/UndefHandler.hpp index 57a7d22..7affa39 100644 --- a/lib/preprocessor/directives_processor/handlers/UndefHandler.hpp +++ b/lib/preprocessor/directives_processor/handlers/UndefHandler.hpp @@ -2,7 +2,7 @@ #define PREPROCESSOR_UNDEF_HANDLER_HPP_ #include "DirectiveHandler.hpp" -#include "lib/lexer/tokens/Token.hpp" +#include "lib/tokens/Token.hpp" namespace ovum::compiler::preprocessor { diff --git a/lib/preprocessor/import_processor/TokenImportProcessor.cpp b/lib/preprocessor/import_processor/TokenImportProcessor.cpp index 4ae9f18..688068d 100644 --- a/lib/preprocessor/import_processor/TokenImportProcessor.cpp +++ b/lib/preprocessor/import_processor/TokenImportProcessor.cpp @@ -124,7 +124,7 @@ std::expected TokenImportProcessor::GatherDependencies( std::string content_str = std::move(content_result.value()); std::string_view content_view(content_str); - Lexer lexer(content_view, false); + lexer::Lexer lexer(content_view, false); std::vector raw_tokens; try { diff --git a/lib/preprocessor/import_processor/TokenImportProcessor.hpp b/lib/preprocessor/import_processor/TokenImportProcessor.hpp index ad75bfc..e5fbf0a 100644 --- a/lib/preprocessor/import_processor/TokenImportProcessor.hpp +++ b/lib/preprocessor/import_processor/TokenImportProcessor.hpp @@ -10,9 +10,9 @@ #include #include "FileGraph.hpp" -#include "lib/lexer/tokens/Token.hpp" #include "lib/preprocessor/PreprocessorError.hpp" #include "lib/preprocessor/TokenProcessor.hpp" +#include "lib/tokens/Token.hpp" namespace ovum::compiler::preprocessor { diff --git a/lib/tokens/CMakeLists.txt b/lib/tokens/CMakeLists.txt new file mode 100644 index 0000000..c56f470 --- /dev/null +++ b/lib/tokens/CMakeLists.txt @@ -0,0 +1,28 @@ +add_library(tokens STATIC + CommentToken.cpp + EofToken.cpp + IdentToken.cpp + KeywordToken.cpp + LiteralToken.cpp + NewlineToken.cpp + OperatorToken.cpp + PunctToken.cpp + TokenFactory.cpp + TokenPosition.cpp + values/BoolValue.cpp + values/CharValue.cpp + values/FloatValue.cpp + values/IntValue.cpp + values/StringValue.cpp +) + +target_include_directories(tokens + PUBLIC + ${CMAKE_SOURCE_DIR} +) + +target_include_directories(tokens + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR} +) + diff --git a/lib/lexer/tokens/CommentToken.cpp b/lib/tokens/CommentToken.cpp similarity index 95% rename from lib/lexer/tokens/CommentToken.cpp rename to lib/tokens/CommentToken.cpp index 27397b1..8f88c63 100644 --- a/lib/lexer/tokens/CommentToken.cpp +++ b/lib/tokens/CommentToken.cpp @@ -2,6 +2,8 @@ #include +namespace ovum { + CommentToken::CommentToken(std::string txt, const TokenPosition& position) : text_(std::move(txt)), position_(position) { } @@ -31,3 +33,5 @@ std::string CommentToken::ToString() const { const TokenPosition& CommentToken::GetPosition() const noexcept { return position_; } + +} // namespace ovum diff --git a/lib/lexer/tokens/CommentToken.hpp b/lib/tokens/CommentToken.hpp similarity index 81% rename from lib/lexer/tokens/CommentToken.hpp rename to lib/tokens/CommentToken.hpp index 319da04..c3dec6b 100644 --- a/lib/lexer/tokens/CommentToken.hpp +++ b/lib/tokens/CommentToken.hpp @@ -1,11 +1,13 @@ -#ifndef COMMENTTOKEN_HPP_ -#define COMMENTTOKEN_HPP_ +#ifndef TOKENS_COMMENTTOKEN_HPP_ +#define TOKENS_COMMENTTOKEN_HPP_ #include #include "Token.hpp" #include "TokenVisitor.hpp" +namespace ovum { + class CommentToken final : public Token { public: CommentToken(std::string txt, const TokenPosition& position); @@ -27,4 +29,6 @@ class CommentToken final : public Token { TokenPosition position_; }; -#endif // COMMENTTOKEN_HPP_ +} // namespace ovum + +#endif // TOKENS_COMMENTTOKEN_HPP_ diff --git a/lib/lexer/tokens/EofToken.cpp b/lib/tokens/EofToken.cpp similarity index 94% rename from lib/lexer/tokens/EofToken.cpp rename to lib/tokens/EofToken.cpp index de3ad16..f357e10 100644 --- a/lib/lexer/tokens/EofToken.cpp +++ b/lib/tokens/EofToken.cpp @@ -2,6 +2,8 @@ #include +namespace ovum { + EofToken::EofToken(const TokenPosition& position) : position_(position) { } @@ -30,3 +32,5 @@ std::string EofToken::ToString() const { const TokenPosition& EofToken::GetPosition() const noexcept { return position_; } + +} // namespace ovum diff --git a/lib/lexer/tokens/EofToken.hpp b/lib/tokens/EofToken.hpp similarity index 81% rename from lib/lexer/tokens/EofToken.hpp rename to lib/tokens/EofToken.hpp index d6cefe7..a36e906 100644 --- a/lib/lexer/tokens/EofToken.hpp +++ b/lib/tokens/EofToken.hpp @@ -1,11 +1,13 @@ -#ifndef EOFTOKEN_HPP_ -#define EOFTOKEN_HPP_ +#ifndef TOKENS_EOFTOKEN_HPP_ +#define TOKENS_EOFTOKEN_HPP_ #include #include "Token.hpp" #include "TokenVisitor.hpp" +namespace ovum { + class EofToken final : public Token { public: EofToken(const TokenPosition& position); @@ -26,4 +28,6 @@ class EofToken final : public Token { TokenPosition position_; }; -#endif // EOFTOKEN_HPP_ +} // namespace ovum + +#endif // TOKENS_EOFTOKEN_HPP_ diff --git a/lib/lexer/tokens/IdentToken.cpp b/lib/tokens/IdentToken.cpp similarity index 95% rename from lib/lexer/tokens/IdentToken.cpp rename to lib/tokens/IdentToken.cpp index 892eff3..98f1404 100644 --- a/lib/lexer/tokens/IdentToken.cpp +++ b/lib/tokens/IdentToken.cpp @@ -2,6 +2,8 @@ #include +namespace ovum { + IdentToken::IdentToken(std::string lex, const TokenPosition& position) : lexeme_(std::move(lex)), position_(position) { } @@ -34,3 +36,5 @@ const TokenPosition& IdentToken::GetPosition() const noexcept { const std::string& IdentToken::GetName() const noexcept { return lexeme_; } + +} // namespace ovum diff --git a/lib/lexer/tokens/IdentToken.hpp b/lib/tokens/IdentToken.hpp similarity index 83% rename from lib/lexer/tokens/IdentToken.hpp rename to lib/tokens/IdentToken.hpp index e3709ff..0274a5c 100644 --- a/lib/lexer/tokens/IdentToken.hpp +++ b/lib/tokens/IdentToken.hpp @@ -1,11 +1,13 @@ -#ifndef IDENTTOKEN_HPP_ -#define IDENTTOKEN_HPP_ +#ifndef TOKENS_IDENTTOKEN_HPP_ +#define TOKENS_IDENTTOKEN_HPP_ #include #include "Token.hpp" #include "TokenVisitor.hpp" +namespace ovum { + class IdentToken final : public Token { public: IdentToken(std::string lex, const TokenPosition& position); @@ -29,4 +31,6 @@ class IdentToken final : public Token { TokenPosition position_; }; -#endif // IDENTTOKEN_HPP_ +} // namespace ovum + +#endif // TOKENS_IDENTTOKEN_HPP_ diff --git a/lib/lexer/tokens/KeywordToken.cpp b/lib/tokens/KeywordToken.cpp similarity index 95% rename from lib/lexer/tokens/KeywordToken.cpp rename to lib/tokens/KeywordToken.cpp index 82cd8d1..ca7d610 100644 --- a/lib/lexer/tokens/KeywordToken.cpp +++ b/lib/tokens/KeywordToken.cpp @@ -2,6 +2,8 @@ #include +namespace ovum { + KeywordToken::KeywordToken(std::string lex, const TokenPosition& position) : lexeme_(std::move(lex)), position_(position) { } @@ -31,3 +33,5 @@ std::string KeywordToken::ToString() const { const TokenPosition& KeywordToken::GetPosition() const noexcept { return position_; } + +} // namespace ovum diff --git a/lib/lexer/tokens/KeywordToken.hpp b/lib/tokens/KeywordToken.hpp similarity index 81% rename from lib/lexer/tokens/KeywordToken.hpp rename to lib/tokens/KeywordToken.hpp index 7860e5f..05354c2 100644 --- a/lib/lexer/tokens/KeywordToken.hpp +++ b/lib/tokens/KeywordToken.hpp @@ -1,11 +1,13 @@ -#ifndef KEYWORDTOKEN_HPP_ -#define KEYWORDTOKEN_HPP_ +#ifndef TOKENS_KEYWORDTOKEN_HPP_ +#define TOKENS_KEYWORDTOKEN_HPP_ #include #include "Token.hpp" #include "TokenVisitor.hpp" +namespace ovum { + class KeywordToken final : public Token { public: KeywordToken(std::string lex, const TokenPosition& position); @@ -27,4 +29,6 @@ class KeywordToken final : public Token { TokenPosition position_; }; -#endif // KEYWORDTOKEN_HPP_ +} // namespace ovum + +#endif // TOKENS_KEYWORDTOKEN_HPP_ diff --git a/lib/lexer/tokens/LiteralToken.cpp b/lib/tokens/LiteralToken.cpp similarity index 96% rename from lib/lexer/tokens/LiteralToken.cpp rename to lib/tokens/LiteralToken.cpp index a18c863..0ed75d7 100644 --- a/lib/lexer/tokens/LiteralToken.cpp +++ b/lib/tokens/LiteralToken.cpp @@ -3,6 +3,8 @@ #include #include +namespace ovum { + LiteralToken::LiteralToken(std::string rawLexeme, std::unique_ptr value, const TokenPosition& position) : lexeme_(std::move(rawLexeme)), value_(std::move(value)), position_(position) { } @@ -47,3 +49,5 @@ std::string LiteralToken::ToString() const { const TokenPosition& LiteralToken::GetPosition() const noexcept { return position_; } + +} // namespace ovum diff --git a/lib/lexer/tokens/LiteralToken.hpp b/lib/tokens/LiteralToken.hpp similarity index 82% rename from lib/lexer/tokens/LiteralToken.hpp rename to lib/tokens/LiteralToken.hpp index 2b30d17..9a475cc 100644 --- a/lib/lexer/tokens/LiteralToken.hpp +++ b/lib/tokens/LiteralToken.hpp @@ -1,12 +1,14 @@ -#ifndef LITERALTOKEN_HPP_ -#define LITERALTOKEN_HPP_ +#ifndef TOKENS_LITERALTOKEN_HPP_ +#define TOKENS_LITERALTOKEN_HPP_ #include #include #include "Token.hpp" #include "TokenVisitor.hpp" -#include "lib/lexer/values/Value.hpp" +#include "values/Value.hpp" + +namespace ovum { class LiteralToken final : public Token { public: @@ -34,4 +36,6 @@ class LiteralToken final : public Token { TokenPosition position_; }; -#endif // LITERALTOKEN_HPP_ +} // namespace ovum + +#endif // TOKENS_LITERALTOKEN_HPP_ diff --git a/lib/lexer/tokens/NewlineToken.cpp b/lib/tokens/NewlineToken.cpp similarity index 95% rename from lib/lexer/tokens/NewlineToken.cpp rename to lib/tokens/NewlineToken.cpp index bcbdcaa..99d96e3 100644 --- a/lib/lexer/tokens/NewlineToken.cpp +++ b/lib/tokens/NewlineToken.cpp @@ -2,6 +2,8 @@ #include +namespace ovum { + NewlineToken::NewlineToken(const TokenPosition& position) : lexeme_("\\n"), position_(position) { } @@ -30,3 +32,5 @@ std::string NewlineToken::ToString() const { const TokenPosition& NewlineToken::GetPosition() const noexcept { return position_; } + +} // namespace ovum diff --git a/lib/lexer/tokens/NewlineToken.hpp b/lib/tokens/NewlineToken.hpp similarity index 81% rename from lib/lexer/tokens/NewlineToken.hpp rename to lib/tokens/NewlineToken.hpp index 02686b4..72f63fa 100644 --- a/lib/lexer/tokens/NewlineToken.hpp +++ b/lib/tokens/NewlineToken.hpp @@ -1,11 +1,13 @@ -#ifndef NEWLINETOKEN_HPP_ -#define NEWLINETOKEN_HPP_ +#ifndef TOKENS_NEWLINETOKEN_HPP_ +#define TOKENS_NEWLINETOKEN_HPP_ #include #include "Token.hpp" #include "TokenVisitor.hpp" +namespace ovum { + class NewlineToken final : public Token { public: NewlineToken(const TokenPosition& position); @@ -27,4 +29,6 @@ class NewlineToken final : public Token { TokenPosition position_; }; -#endif // NEWLINETOKEN_HPP_ +} // namespace ovum + +#endif // TOKENS_NEWLINETOKEN_HPP_ diff --git a/lib/lexer/tokens/OperatorToken.cpp b/lib/tokens/OperatorToken.cpp similarity index 95% rename from lib/lexer/tokens/OperatorToken.cpp rename to lib/tokens/OperatorToken.cpp index 32be80d..391a527 100644 --- a/lib/lexer/tokens/OperatorToken.cpp +++ b/lib/tokens/OperatorToken.cpp @@ -2,6 +2,8 @@ #include +namespace ovum { + OperatorToken::OperatorToken(std::string op, const TokenPosition& position) : lexeme_(std::move(op)), position_(position) { } @@ -31,3 +33,5 @@ std::string OperatorToken::ToString() const { const TokenPosition& OperatorToken::GetPosition() const noexcept { return position_; } + +} // namespace ovum diff --git a/lib/lexer/tokens/OperatorToken.hpp b/lib/tokens/OperatorToken.hpp similarity index 81% rename from lib/lexer/tokens/OperatorToken.hpp rename to lib/tokens/OperatorToken.hpp index 5f82a6b..0e6dede 100644 --- a/lib/lexer/tokens/OperatorToken.hpp +++ b/lib/tokens/OperatorToken.hpp @@ -1,10 +1,12 @@ -#ifndef OPERATORTOKEN_HPP_ -#define OPERATORTOKEN_HPP_ +#ifndef TOKENS_OPERATORTOKEN_HPP_ +#define TOKENS_OPERATORTOKEN_HPP_ #include #include "Token.hpp" +namespace ovum { + class TokenVisitor; class OperatorToken final : public Token { @@ -28,4 +30,6 @@ class OperatorToken final : public Token { TokenPosition position_; }; -#endif // OPERATORTOKEN_HPP_ +} // namespace ovum + +#endif // TOKENS_OPERATORTOKEN_HPP_ diff --git a/lib/lexer/tokens/PunctToken.cpp b/lib/tokens/PunctToken.cpp similarity index 95% rename from lib/lexer/tokens/PunctToken.cpp rename to lib/tokens/PunctToken.cpp index 47348cc..9815f5e 100644 --- a/lib/lexer/tokens/PunctToken.cpp +++ b/lib/tokens/PunctToken.cpp @@ -1,6 +1,8 @@ #include #include "PunctToken.hpp" +namespace ovum { + PunctToken::PunctToken(char ch, const TokenPosition& position) : lexeme_(1, ch), position_(position) { } @@ -33,3 +35,5 @@ std::string PunctToken::ToString() const { const TokenPosition& PunctToken::GetPosition() const noexcept { return position_; } + +} // namespace ovum diff --git a/lib/lexer/tokens/PunctToken.hpp b/lib/tokens/PunctToken.hpp similarity index 83% rename from lib/lexer/tokens/PunctToken.hpp rename to lib/tokens/PunctToken.hpp index d7feafe..89de415 100644 --- a/lib/lexer/tokens/PunctToken.hpp +++ b/lib/tokens/PunctToken.hpp @@ -1,11 +1,13 @@ -#ifndef PUNCTTOKEN_HPP_ -#define PUNCTTOKEN_HPP_ +#ifndef TOKENS_PUNCTTOKEN_HPP_ +#define TOKENS_PUNCTTOKEN_HPP_ #include #include "Token.hpp" #include "TokenVisitor.hpp" +namespace ovum { + class PunctToken final : public Token { public: PunctToken(char ch, const TokenPosition& position); @@ -29,4 +31,6 @@ class PunctToken final : public Token { TokenPosition position_; }; -#endif // PUNCTTOKEN_HPP_ +} // namespace ovum + +#endif // TOKENS_PUNCTTOKEN_HPP_ diff --git a/lib/lexer/tokens/Token.hpp b/lib/tokens/Token.hpp similarity index 84% rename from lib/lexer/tokens/Token.hpp rename to lib/tokens/Token.hpp index 499b313..a1a85bf 100644 --- a/lib/lexer/tokens/Token.hpp +++ b/lib/tokens/Token.hpp @@ -1,5 +1,5 @@ -#ifndef TOKEN_HPP_ -#define TOKEN_HPP_ +#ifndef TOKENS_TOKEN_HPP_ +#define TOKENS_TOKEN_HPP_ #include #include @@ -7,6 +7,8 @@ #include "TokenPosition.hpp" #include "TokenVisitor.hpp" +namespace ovum { + class Token { // NOLINT(cppcoreguidelines-special-member-functions) public: virtual ~Token() = default; @@ -26,4 +28,6 @@ class Token { // NOLINT(cppcoreguidelines-special-member-functions) using TokenPtr = std::shared_ptr; -#endif // TOKEN_HPP_ +} // namespace ovum + +#endif // TOKENS_TOKEN_HPP_ diff --git a/lib/lexer/tokens/TokenFactory.cpp b/lib/tokens/TokenFactory.cpp similarity index 90% rename from lib/lexer/tokens/TokenFactory.cpp rename to lib/tokens/TokenFactory.cpp index aca2384..114b77d 100644 --- a/lib/lexer/tokens/TokenFactory.cpp +++ b/lib/tokens/TokenFactory.cpp @@ -8,11 +8,13 @@ #include "NewlineToken.hpp" #include "OperatorToken.hpp" #include "PunctToken.hpp" -#include "lib/lexer/values/BoolValue.hpp" -#include "lib/lexer/values/CharValue.hpp" -#include "lib/lexer/values/FloatValue.hpp" -#include "lib/lexer/values/IntValue.hpp" -#include "lib/lexer/values/StringValue.hpp" +#include "values/BoolValue.hpp" +#include "values/CharValue.hpp" +#include "values/FloatValue.hpp" +#include "values/IntValue.hpp" +#include "values/StringValue.hpp" + +namespace ovum { namespace TokenFactory { @@ -69,4 +71,6 @@ TokenPtr MakeBoolLiteral(std::string raw, bool b, int32_t line, int32_t col) { return std::make_shared(std::move(raw), std::make_unique(b), TokenPosition(line, col)); } -}; // namespace TokenFactory +} // namespace TokenFactory + +} // namespace ovum diff --git a/lib/lexer/tokens/TokenFactory.hpp b/lib/tokens/TokenFactory.hpp similarity index 81% rename from lib/lexer/tokens/TokenFactory.hpp rename to lib/tokens/TokenFactory.hpp index 24f460c..e2501ae 100644 --- a/lib/lexer/tokens/TokenFactory.hpp +++ b/lib/tokens/TokenFactory.hpp @@ -1,11 +1,11 @@ -#ifndef TOKENFACTORY_HPP_ -#define TOKENFACTORY_HPP_ +#ifndef TOKENS_TOKENFACTORY_HPP_ +#define TOKENS_TOKENFACTORY_HPP_ #include #include "Token.hpp" -namespace TokenFactory { +namespace ovum::TokenFactory { // NOLINT(readability-identifier-naming) TokenPtr MakeIdent(std::string lex, int32_t line, int32_t col); @@ -33,6 +33,6 @@ TokenPtr MakeCharLiteral(std::string raw, char c, int32_t line, int32_t col); TokenPtr MakeBoolLiteral(std::string raw, bool b, int32_t line, int32_t col); -}; // namespace TokenFactory +} // namespace ovum::TokenFactory -#endif // TOKENFACTORY_HPP_ +#endif // TOKENS_TOKENFACTORY_HPP_ diff --git a/lib/lexer/tokens/TokenPosition.cpp b/lib/tokens/TokenPosition.cpp similarity index 87% rename from lib/lexer/tokens/TokenPosition.cpp rename to lib/tokens/TokenPosition.cpp index 2c9dcec..4d8da5c 100644 --- a/lib/lexer/tokens/TokenPosition.cpp +++ b/lib/tokens/TokenPosition.cpp @@ -1,5 +1,7 @@ #include "TokenPosition.hpp" +namespace ovum { + TokenPosition::TokenPosition(int32_t line, int32_t column) : line_(line), column_(column) { } @@ -10,3 +12,5 @@ int32_t TokenPosition::GetLine() const noexcept { int32_t TokenPosition::GetColumn() const noexcept { return column_; } + +} // namespace ovum diff --git a/lib/lexer/tokens/TokenPosition.hpp b/lib/tokens/TokenPosition.hpp similarity index 63% rename from lib/lexer/tokens/TokenPosition.hpp rename to lib/tokens/TokenPosition.hpp index 2b2be92..9a27154 100644 --- a/lib/lexer/tokens/TokenPosition.hpp +++ b/lib/tokens/TokenPosition.hpp @@ -1,8 +1,10 @@ -#ifndef TOKENPOSITION_HPP_ -#define TOKENPOSITION_HPP_ +#ifndef TOKENS_TOKENPOSITION_HPP_ +#define TOKENS_TOKENPOSITION_HPP_ #include +namespace ovum { + class TokenPosition { public: TokenPosition(int32_t line, int32_t column); @@ -16,4 +18,6 @@ class TokenPosition { int32_t column_; }; -#endif // TOKENPOSITION_HPP_ +} // namespace ovum + +#endif // TOKENS_TOKENPOSITION_HPP_ diff --git a/lib/lexer/tokens/TokenVisitor.hpp b/lib/tokens/TokenVisitor.hpp similarity index 82% rename from lib/lexer/tokens/TokenVisitor.hpp rename to lib/tokens/TokenVisitor.hpp index 323801b..c721c22 100644 --- a/lib/lexer/tokens/TokenVisitor.hpp +++ b/lib/tokens/TokenVisitor.hpp @@ -1,5 +1,7 @@ -#ifndef TOKENVISITOR_HPP_ -#define TOKENVISITOR_HPP_ +#ifndef TOKENS_TOKENVISITOR_HPP_ +#define TOKENS_TOKENVISITOR_HPP_ + +namespace ovum { class EofToken; class CommentToken; @@ -31,4 +33,6 @@ class TokenVisitor { // NOLINT(cppcoreguidelines-special-member-functions) virtual void Visit(const EofToken& t) = 0; }; -#endif // TOKENVISITOR_HPP_ +} // namespace ovum + +#endif // TOKENS_TOKENVISITOR_HPP_ diff --git a/lib/lexer/values/BoolValue.cpp b/lib/tokens/values/BoolValue.cpp similarity index 88% rename from lib/lexer/values/BoolValue.cpp rename to lib/tokens/values/BoolValue.cpp index 92fdf05..d60f1a0 100644 --- a/lib/lexer/values/BoolValue.cpp +++ b/lib/tokens/values/BoolValue.cpp @@ -1,5 +1,7 @@ #include "BoolValue.hpp" +namespace ovum { + BoolValue::BoolValue(bool b) : v(b) { } @@ -14,3 +16,5 @@ std::string BoolValue::ToString() const { std::string BoolValue::GetTypeName() const { return "Bool"; } + +} // namespace ovum diff --git a/lib/lexer/values/BoolValue.hpp b/lib/tokens/values/BoolValue.hpp similarity index 71% rename from lib/lexer/values/BoolValue.hpp rename to lib/tokens/values/BoolValue.hpp index f51f3a9..fca95f9 100644 --- a/lib/lexer/values/BoolValue.hpp +++ b/lib/tokens/values/BoolValue.hpp @@ -1,11 +1,13 @@ -#ifndef BOOLVALUE_HPP_ -#define BOOLVALUE_HPP_ +#ifndef TOKENS_BOOLVALUE_HPP_ +#define TOKENS_BOOLVALUE_HPP_ #include #include #include "Value.hpp" +namespace ovum { + class BoolValue : public Value { public: explicit BoolValue(bool b); @@ -20,4 +22,6 @@ class BoolValue : public Value { bool v; }; -#endif // BOOLVALUE_HPP_ +} // namespace ovum + +#endif // TOKENS_BOOLVALUE_HPP_ diff --git a/lib/lexer/values/CharValue.cpp b/lib/tokens/values/CharValue.cpp similarity index 88% rename from lib/lexer/values/CharValue.cpp rename to lib/tokens/values/CharValue.cpp index 4c7f584..78146ef 100644 --- a/lib/lexer/values/CharValue.cpp +++ b/lib/tokens/values/CharValue.cpp @@ -1,5 +1,7 @@ #include "CharValue.hpp" +namespace ovum { + CharValue::CharValue(char c) : v(c) { } @@ -14,3 +16,5 @@ std::string CharValue::ToString() const { std::string CharValue::GetTypeName() const { return "Char"; } + +} // namespace ovum diff --git a/lib/lexer/values/CharValue.hpp b/lib/tokens/values/CharValue.hpp similarity index 71% rename from lib/lexer/values/CharValue.hpp rename to lib/tokens/values/CharValue.hpp index e111327..263fca8 100644 --- a/lib/lexer/values/CharValue.hpp +++ b/lib/tokens/values/CharValue.hpp @@ -1,11 +1,13 @@ -#ifndef CHARVALUE_HPP_ -#define CHARVALUE_HPP_ +#ifndef TOKENS_CHARVALUE_HPP_ +#define TOKENS_CHARVALUE_HPP_ #include #include #include "Value.hpp" +namespace ovum { + class CharValue : public Value { public: explicit CharValue(char c); @@ -20,4 +22,6 @@ class CharValue : public Value { char v; }; -#endif // CHARVALUE_HPP_ +} // namespace ovum + +#endif // TOKENS_CHARVALUE_HPP_ diff --git a/lib/lexer/values/FloatValue.cpp b/lib/tokens/values/FloatValue.cpp similarity index 90% rename from lib/lexer/values/FloatValue.cpp rename to lib/tokens/values/FloatValue.cpp index a9c790f..686ca23 100644 --- a/lib/lexer/values/FloatValue.cpp +++ b/lib/tokens/values/FloatValue.cpp @@ -2,6 +2,8 @@ #include +namespace ovum { + FloatValue::FloatValue(double x) : v(x) { } @@ -18,3 +20,5 @@ std::string FloatValue::ToString() const { std::string FloatValue::GetTypeName() const { return "Float"; } + +} // namespace ovum diff --git a/lib/lexer/values/FloatValue.hpp b/lib/tokens/values/FloatValue.hpp similarity index 71% rename from lib/lexer/values/FloatValue.hpp rename to lib/tokens/values/FloatValue.hpp index 77c8e1c..92b860f 100644 --- a/lib/lexer/values/FloatValue.hpp +++ b/lib/tokens/values/FloatValue.hpp @@ -1,11 +1,13 @@ -#ifndef FLOATVALUE_HPP_ -#define FLOATVALUE_HPP_ +#ifndef TOKENS_FLOATVALUE_HPP_ +#define TOKENS_FLOATVALUE_HPP_ #include #include #include "Value.hpp" +namespace ovum { + class FloatValue : public Value { public: explicit FloatValue(double x); @@ -20,4 +22,6 @@ class FloatValue : public Value { double v; }; -#endif // FLOATVALUE_HPP_ +} // namespace ovum + +#endif // TOKENS_FLOATVALUE_HPP_ diff --git a/lib/lexer/values/IntValue.cpp b/lib/tokens/values/IntValue.cpp similarity index 88% rename from lib/lexer/values/IntValue.cpp rename to lib/tokens/values/IntValue.cpp index d4e21df..3cabf55 100644 --- a/lib/lexer/values/IntValue.cpp +++ b/lib/tokens/values/IntValue.cpp @@ -1,5 +1,7 @@ #include "IntValue.hpp" +namespace ovum { + IntValue::IntValue(int64_t x) : v(x) { } @@ -14,3 +16,5 @@ std::string IntValue::ToString() const { std::string IntValue::GetTypeName() const { return "Int"; } + +} // namespace ovum diff --git a/lib/lexer/values/IntValue.hpp b/lib/tokens/values/IntValue.hpp similarity index 73% rename from lib/lexer/values/IntValue.hpp rename to lib/tokens/values/IntValue.hpp index 45a3194..3df8612 100644 --- a/lib/lexer/values/IntValue.hpp +++ b/lib/tokens/values/IntValue.hpp @@ -1,5 +1,5 @@ -#ifndef INTVALUE_HPP_ -#define INTVALUE_HPP_ +#ifndef TOKENS_INTVALUE_HPP_ +#define TOKENS_INTVALUE_HPP_ #include #include @@ -7,6 +7,8 @@ #include "Value.hpp" +namespace ovum { + class IntValue : public Value { public: explicit IntValue(int64_t x); @@ -21,4 +23,6 @@ class IntValue : public Value { int64_t v; }; -#endif // INTVALUE_HPP_ +} // namespace ovum + +#endif // TOKENS_INTVALUE_HPP_ diff --git a/lib/lexer/values/StringValue.cpp b/lib/tokens/values/StringValue.cpp similarity index 89% rename from lib/lexer/values/StringValue.cpp rename to lib/tokens/values/StringValue.cpp index a5dbbbf..75f84f0 100644 --- a/lib/lexer/values/StringValue.cpp +++ b/lib/tokens/values/StringValue.cpp @@ -2,6 +2,8 @@ #include +namespace ovum { + StringValue::StringValue(std::string s) : v(std::move(s)) { } @@ -16,3 +18,5 @@ std::string StringValue::ToString() const { std::string StringValue::GetTypeName() const { return "String"; } + +} // namespace ovum diff --git a/lib/lexer/values/StringValue.hpp b/lib/tokens/values/StringValue.hpp similarity index 71% rename from lib/lexer/values/StringValue.hpp rename to lib/tokens/values/StringValue.hpp index c4d04ce..8649b41 100644 --- a/lib/lexer/values/StringValue.hpp +++ b/lib/tokens/values/StringValue.hpp @@ -1,11 +1,13 @@ -#ifndef STRINGVALUE_HPP_ -#define STRINGVALUE_HPP_ +#ifndef TOKENS_STRINGVALUE_HPP_ +#define TOKENS_STRINGVALUE_HPP_ #include #include #include "Value.hpp" +namespace ovum { + class StringValue : public Value { public: explicit StringValue(std::string s); @@ -20,4 +22,6 @@ class StringValue : public Value { std::string v; }; -#endif // STRINGVALUE_HPP_ +} // namespace ovum + +#endif // TOKENS_STRINGVALUE_HPP_ diff --git a/lib/lexer/values/Value.hpp b/lib/tokens/values/Value.hpp similarity index 73% rename from lib/lexer/values/Value.hpp rename to lib/tokens/values/Value.hpp index b6ad569..d4c572c 100644 --- a/lib/lexer/values/Value.hpp +++ b/lib/tokens/values/Value.hpp @@ -1,9 +1,11 @@ -#ifndef VALUE_HPP_ -#define VALUE_HPP_ +#ifndef TOKENS_VALUE_HPP_ +#define TOKENS_VALUE_HPP_ #include #include +namespace ovum { + class Value { // NOLINT(cppcoreguidelines-special-member-functions) public: virtual ~Value() = default; @@ -15,4 +17,6 @@ class Value { // NOLINT(cppcoreguidelines-special-member-functions) [[nodiscard]] virtual std::string GetTypeName() const = 0; }; -#endif // VALUE_HPP_ +} // namespace ovum + +#endif // TOKENS_VALUE_HPP_ diff --git a/tests/lexer_big_programs_tests.cpp b/tests/lexer_big_programs_tests.cpp index 574bd12..67c5749 100644 --- a/tests/lexer_big_programs_tests.cpp +++ b/tests/lexer_big_programs_tests.cpp @@ -4,6 +4,8 @@ #include "lib/lexer/Lexer.hpp" #include "test_suites/LexerUnitTestSuite.hpp" +using ovum::compiler::lexer::Lexer; + TEST(LexerUnitTestSuite, ExampleFundamentals) { const std::string src = R"OVUM(fun ExampleFundamentals(): Void { val i: int = 42 diff --git a/tests/lexer_tests.cpp b/tests/lexer_tests.cpp index f6a9fb4..26a88aa 100644 --- a/tests/lexer_tests.cpp +++ b/tests/lexer_tests.cpp @@ -4,6 +4,8 @@ #include "lib/lexer/Lexer.hpp" #include "test_suites/LexerUnitTestSuite.hpp" +using ovum::compiler::lexer::Lexer; + TEST(LexerUnitTestSuite, EmptyString) { const std::string src = ""; Lexer lexer(src); diff --git a/tests/test_suites/LexerUnitTestSuite.cpp b/tests/test_suites/LexerUnitTestSuite.cpp index 86c4e31..9591cf4 100644 --- a/tests/test_suites/LexerUnitTestSuite.cpp +++ b/tests/test_suites/LexerUnitTestSuite.cpp @@ -9,7 +9,7 @@ void LexerUnitTestSuite::TearDown() { } std::vector> LexerUnitTestSuite::ExtractLexemesAndTypes( - const std::vector& tokens) { + const std::vector& tokens) { std::vector> out; if (tokens.empty()) return out; diff --git a/tests/test_suites/LexerUnitTestSuite.hpp b/tests/test_suites/LexerUnitTestSuite.hpp index eb5a9ff..4c0c3fc 100644 --- a/tests/test_suites/LexerUnitTestSuite.hpp +++ b/tests/test_suites/LexerUnitTestSuite.hpp @@ -2,10 +2,12 @@ #define OVUMC_LEXERUNITTESTSUITE_HPP #include -#include "lib/lexer/tokens/Token.hpp" +#include "lib/tokens/Token.hpp" struct LexerUnitTestSuite : public testing::Test { - static std::vector> ExtractLexemesAndTypes(const std::vector& tokens); +public: + static std::vector> ExtractLexemesAndTypes( + const std::vector& tokens); static void PrintLexemesAndTypes(const std::vector>& items); diff --git a/tests/test_suites/PreprocessorUnitTestSuite.cpp b/tests/test_suites/PreprocessorUnitTestSuite.cpp index 030ea9d..cda1bb2 100644 --- a/tests/test_suites/PreprocessorUnitTestSuite.cpp +++ b/tests/test_suites/PreprocessorUnitTestSuite.cpp @@ -76,7 +76,7 @@ std::expected, std::string> PreprocessorUnitTestSuite::Tok std::string content((std::istreambuf_iterator(file)), std::istreambuf_iterator()); - Lexer lexer(content, false); + ovum::compiler::lexer::Lexer lexer(content, false); auto tokens_result = lexer.Tokenize(); return tokens_result; diff --git a/tests/test_suites/PreprocessorUnitTestSuite.hpp b/tests/test_suites/PreprocessorUnitTestSuite.hpp index 533df9a..c56556e 100644 --- a/tests/test_suites/PreprocessorUnitTestSuite.hpp +++ b/tests/test_suites/PreprocessorUnitTestSuite.hpp @@ -6,8 +6,8 @@ #include #include -#include "lib/lexer/tokens/Token.hpp" #include "lib/preprocessor/PreprocessorError.hpp" +#include "lib/tokens/Token.hpp" namespace ovum::compiler::preprocessor { diff --git a/tests/test_suites/ProjectIntegrationTestSuite.hpp b/tests/test_suites/ProjectIntegrationTestSuite.hpp index 0ecab1e..1ac983e 100644 --- a/tests/test_suites/ProjectIntegrationTestSuite.hpp +++ b/tests/test_suites/ProjectIntegrationTestSuite.hpp @@ -6,6 +6,7 @@ #include struct ProjectIntegrationTestSuite : public testing::Test { // special test structure +public: const std::string kTemporaryDirectoryName = "./gtest_tmp"; void SetUp() override; // method that is called at the beginning of every test From 0e3c7486ca9d7e163fe5e35946d2c37c98c1a541 Mon Sep 17 00:00:00 2001 From: bialger Date: Fri, 7 Nov 2025 19:42:01 +0300 Subject: [PATCH 2/3] refactor: update Lexer and Handler classes to use std::expected for error handling in tokenization --- lib/lexer/Lexer.cpp | 10 +- lib/lexer/Lexer.hpp | 4 +- lib/lexer/handlers/CharHandler.cpp | 10 +- lib/lexer/handlers/CharHandler.hpp | 2 +- lib/lexer/handlers/ColonHandler.cpp | 2 +- lib/lexer/handlers/ColonHandler.hpp | 2 +- lib/lexer/handlers/DefaultHandler.cpp | 4 +- lib/lexer/handlers/DefaultHandler.hpp | 2 +- lib/lexer/handlers/DotCompositeHandler.cpp | 14 +- lib/lexer/handlers/DotCompositeHandler.hpp | 2 +- lib/lexer/handlers/Handler.hpp | 4 +- lib/lexer/handlers/IdentifierHandler.cpp | 4 +- lib/lexer/handlers/IdentifierHandler.hpp | 2 +- lib/lexer/handlers/NewlineHandler.cpp | 2 +- lib/lexer/handlers/NewlineHandler.hpp | 2 +- lib/lexer/handlers/NumberHandler.cpp | 118 +++++++---- lib/lexer/handlers/NumberHandler.hpp | 2 +- lib/lexer/handlers/OperatorHandler.cpp | 2 +- lib/lexer/handlers/OperatorHandler.hpp | 2 +- lib/lexer/handlers/PunctHandler.cpp | 2 +- lib/lexer/handlers/PunctHandler.hpp | 2 +- lib/lexer/handlers/SlashHandler.cpp | 4 +- lib/lexer/handlers/SlashHandler.hpp | 2 +- lib/lexer/handlers/StringHandler.cpp | 10 +- lib/lexer/handlers/StringHandler.hpp | 2 +- lib/lexer/handlers/WhitespaceHandler.cpp | 2 +- lib/lexer/handlers/WhitespaceHandler.hpp | 2 +- lib/preprocessor/Preprocessor.cpp | 11 +- .../import_processor/TokenImportProcessor.cpp | 11 +- tests/lexer_big_programs_tests.cpp | 135 +++++++----- tests/lexer_tests.cpp | 193 +++++++++++------- .../test_suites/PreprocessorUnitTestSuite.cpp | 6 +- 32 files changed, 353 insertions(+), 219 deletions(-) diff --git a/lib/lexer/Lexer.cpp b/lib/lexer/Lexer.cpp index ffd59fc..442f63c 100644 --- a/lib/lexer/Lexer.cpp +++ b/lib/lexer/Lexer.cpp @@ -29,7 +29,7 @@ Lexer::Lexer(std::string_view src, bool keep_comments) : wrapper_(src, keep_comments), handlers_(MakeDefaultHandlers()), default_handler_(MakeDefaultHandler()) { } -std::vector Lexer::Tokenize() { +std::expected, LexerError> Lexer::Tokenize() { std::vector tokens; tokens.reserve(kDefaultTokenReserve); @@ -43,7 +43,13 @@ std::vector Lexer::Tokenize() { current_handler = default_handler_.get(); } - OptToken maybe_token = current_handler->Scan(wrapper_); + auto maybe_token_result = current_handler->Scan(wrapper_); + + if (!maybe_token_result) { + return std::unexpected(maybe_token_result.error()); + } + + OptToken maybe_token = maybe_token_result.value(); if (maybe_token && *maybe_token) { tokens.push_back(std::move(*maybe_token)); diff --git a/lib/lexer/Lexer.hpp b/lib/lexer/Lexer.hpp index a7f471c..7e664ef 100644 --- a/lib/lexer/Lexer.hpp +++ b/lib/lexer/Lexer.hpp @@ -3,10 +3,12 @@ #include #include +#include #include #include #include +#include "LexerError.hpp" #include "SourceCodeWrapper.hpp" #include "handlers/Handler.hpp" #include "lib/tokens/Token.hpp" @@ -21,7 +23,7 @@ class Lexer { public: explicit Lexer(std::string_view src, bool keep_comments = false); - std::vector Tokenize(); + std::expected, LexerError> Tokenize(); void SetHandler(unsigned char ch, std::unique_ptr handler); diff --git a/lib/lexer/handlers/CharHandler.cpp b/lib/lexer/handlers/CharHandler.cpp index 47fd764..f2846f5 100644 --- a/lib/lexer/handlers/CharHandler.cpp +++ b/lib/lexer/handlers/CharHandler.cpp @@ -5,7 +5,7 @@ namespace ovum::compiler::lexer { -OptToken CharHandler::Scan(SourceCodeWrapper& wrapper) { +std::expected CharHandler::Scan(SourceCodeWrapper& wrapper) { std::string raw; raw.push_back('\''); char val = '\0'; @@ -32,7 +32,7 @@ OptToken CharHandler::Scan(SourceCodeWrapper& wrapper) { val = '\0'; break; default: - throw LexerError(std::string("Unknown escape in char literal: \\") + e); + return std::unexpected(LexerError(std::string("Unknown escape in char literal: \\") + e)); } } else { char c = wrapper.Advance(); @@ -41,18 +41,18 @@ OptToken CharHandler::Scan(SourceCodeWrapper& wrapper) { } if (wrapper.IsAtEnd()) { - throw LexerError("Unterminated char literal"); + return std::unexpected(LexerError("Unterminated char literal")); } if (wrapper.Peek() == '\n') { - throw LexerError("Newline in char literal"); + return std::unexpected(LexerError("Newline in char literal")); } if (wrapper.Peek() == '\'') { wrapper.Advance(); raw.push_back('\''); } else { - throw LexerError("Too many characters in char literal"); + return std::unexpected(LexerError("Too many characters in char literal")); } return std::make_optional( diff --git a/lib/lexer/handlers/CharHandler.hpp b/lib/lexer/handlers/CharHandler.hpp index fc8baa9..de6981d 100644 --- a/lib/lexer/handlers/CharHandler.hpp +++ b/lib/lexer/handlers/CharHandler.hpp @@ -7,7 +7,7 @@ namespace ovum::compiler::lexer { class CharHandler : public Handler { public: - OptToken Scan(SourceCodeWrapper& wrapper) override; + std::expected Scan(SourceCodeWrapper& wrapper) override; }; } // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/ColonHandler.cpp b/lib/lexer/handlers/ColonHandler.cpp index ef57089..77dc49f 100644 --- a/lib/lexer/handlers/ColonHandler.cpp +++ b/lib/lexer/handlers/ColonHandler.cpp @@ -3,7 +3,7 @@ namespace ovum::compiler::lexer { -OptToken ColonHandler::Scan(SourceCodeWrapper& wrapper) { +std::expected ColonHandler::Scan(SourceCodeWrapper& wrapper) { std::string op; op.push_back(wrapper.CurrentChar()); diff --git a/lib/lexer/handlers/ColonHandler.hpp b/lib/lexer/handlers/ColonHandler.hpp index fbdbb74..602c473 100644 --- a/lib/lexer/handlers/ColonHandler.hpp +++ b/lib/lexer/handlers/ColonHandler.hpp @@ -6,7 +6,7 @@ namespace ovum::compiler::lexer { class ColonHandler : public Handler { public: - OptToken Scan(SourceCodeWrapper& wrapper) override; + std::expected Scan(SourceCodeWrapper& wrapper) override; }; } // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/DefaultHandler.cpp b/lib/lexer/handlers/DefaultHandler.cpp index dbfc99d..8c9c708 100644 --- a/lib/lexer/handlers/DefaultHandler.cpp +++ b/lib/lexer/handlers/DefaultHandler.cpp @@ -4,9 +4,9 @@ namespace ovum::compiler::lexer { -OptToken DefaultHandler::Scan(SourceCodeWrapper& wrapper) { +std::expected DefaultHandler::Scan(SourceCodeWrapper& wrapper) { char c = wrapper.CurrentChar(); - throw LexerError(std::string("Unexpected character: ") + c); + return std::unexpected(LexerError(std::string("Unexpected character: ") + c)); } } // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/DefaultHandler.hpp b/lib/lexer/handlers/DefaultHandler.hpp index 85dc411..860b1a7 100644 --- a/lib/lexer/handlers/DefaultHandler.hpp +++ b/lib/lexer/handlers/DefaultHandler.hpp @@ -7,7 +7,7 @@ namespace ovum::compiler::lexer { class DefaultHandler : public Handler { public: - OptToken Scan(SourceCodeWrapper& wrapper) override; + std::expected Scan(SourceCodeWrapper& wrapper) override; }; } // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/DotCompositeHandler.cpp b/lib/lexer/handlers/DotCompositeHandler.cpp index 7d6a6b1..dd7979a 100644 --- a/lib/lexer/handlers/DotCompositeHandler.cpp +++ b/lib/lexer/handlers/DotCompositeHandler.cpp @@ -7,20 +7,20 @@ DotCompositeHandler::DotCompositeHandler() : punct_(std::make_unique()) { } -OptToken DotCompositeHandler::Scan(SourceCodeWrapper& w) { +std::expected DotCompositeHandler::Scan(SourceCodeWrapper& w) { if (std::isdigit(static_cast(w.Peek()))) { return num_->Scan(w); } - if (auto t = op_->Scan(w)) { - return t; + auto op_result = op_->Scan(w); + if (!op_result) { + return op_result; } - - if (punct_) { - return punct_->Scan(w); + if (op_result.value()) { + return op_result; } - return std::nullopt; + return punct_->Scan(w); } } // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/DotCompositeHandler.hpp b/lib/lexer/handlers/DotCompositeHandler.hpp index ea3297a..301fffc 100644 --- a/lib/lexer/handlers/DotCompositeHandler.hpp +++ b/lib/lexer/handlers/DotCompositeHandler.hpp @@ -12,7 +12,7 @@ class DotCompositeHandler : public Handler { public: DotCompositeHandler(); - OptToken Scan(SourceCodeWrapper& wrapper) override; + std::expected Scan(SourceCodeWrapper& wrapper) override; private: std::unique_ptr num_; diff --git a/lib/lexer/handlers/Handler.hpp b/lib/lexer/handlers/Handler.hpp index b6f4db4..f4475eb 100644 --- a/lib/lexer/handlers/Handler.hpp +++ b/lib/lexer/handlers/Handler.hpp @@ -1,8 +1,10 @@ #ifndef LEXER_HANDLER_HPP_ #define LEXER_HANDLER_HPP_ +#include #include +#include "lib/lexer/LexerError.hpp" #include "lib/lexer/SourceCodeWrapper.hpp" #include "lib/tokens/Token.hpp" @@ -15,7 +17,7 @@ class Handler { // NOLINT(cppcoreguidelines-special-member-functions) public: virtual ~Handler() = default; - virtual OptToken Scan(SourceCodeWrapper& wrapper) = 0; + virtual std::expected Scan(SourceCodeWrapper& wrapper) = 0; }; } // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/IdentifierHandler.cpp b/lib/lexer/handlers/IdentifierHandler.cpp index c1ee429..98da3c9 100644 --- a/lib/lexer/handlers/IdentifierHandler.cpp +++ b/lib/lexer/handlers/IdentifierHandler.cpp @@ -8,7 +8,7 @@ namespace ovum::compiler::lexer { -OptToken IdentifierHandler::Scan(SourceCodeWrapper& wrapper) { +std::expected IdentifierHandler::Scan(SourceCodeWrapper& wrapper) { std::string s; s.push_back(wrapper.CurrentChar()); wrapper.ConsumeWhile(s, [](char ch) { return std::isalnum(static_cast(ch)) || ch == '_'; }); @@ -33,7 +33,7 @@ OptToken IdentifierHandler::Scan(SourceCodeWrapper& wrapper) { } if (s[0] == '#') { - throw LexerError(std::string("Not a keyword, started with #: ") + s); + return std::unexpected(LexerError(std::string("Not a keyword, started with #: ") + s)); } return std::make_optional(TokenFactory::MakeIdent(std::move(s), wrapper.GetLine(), wrapper.GetTokenCol())); diff --git a/lib/lexer/handlers/IdentifierHandler.hpp b/lib/lexer/handlers/IdentifierHandler.hpp index e00a7f0..45f9b8c 100644 --- a/lib/lexer/handlers/IdentifierHandler.hpp +++ b/lib/lexer/handlers/IdentifierHandler.hpp @@ -7,7 +7,7 @@ namespace ovum::compiler::lexer { class IdentifierHandler : public Handler { public: - OptToken Scan(SourceCodeWrapper& wrapper) override; + std::expected Scan(SourceCodeWrapper& wrapper) override; }; } // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/NewlineHandler.cpp b/lib/lexer/handlers/NewlineHandler.cpp index e6ae91a..c537793 100644 --- a/lib/lexer/handlers/NewlineHandler.cpp +++ b/lib/lexer/handlers/NewlineHandler.cpp @@ -4,7 +4,7 @@ namespace ovum::compiler::lexer { -OptToken NewlineHandler::Scan(SourceCodeWrapper& wrapper) { +std::expected NewlineHandler::Scan(SourceCodeWrapper& wrapper) { return std::make_optional(TokenFactory::MakeNewline(wrapper.GetLine() - 1, wrapper.GetTokenCol())); } diff --git a/lib/lexer/handlers/NewlineHandler.hpp b/lib/lexer/handlers/NewlineHandler.hpp index 025df47..b85a71c 100644 --- a/lib/lexer/handlers/NewlineHandler.hpp +++ b/lib/lexer/handlers/NewlineHandler.hpp @@ -7,7 +7,7 @@ namespace ovum::compiler::lexer { class NewlineHandler : public Handler { public: - OptToken Scan(SourceCodeWrapper& wrapper) override; + std::expected Scan(SourceCodeWrapper& wrapper) override; }; } // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/NumberHandler.cpp b/lib/lexer/handlers/NumberHandler.cpp index fe2977c..373b6ac 100644 --- a/lib/lexer/handlers/NumberHandler.cpp +++ b/lib/lexer/handlers/NumberHandler.cpp @@ -31,7 +31,7 @@ inline bool IsIdentStart(char c) noexcept { return std::isalpha(u) != 0 || u == '_'; } -inline bool ParseExponent(SourceCodeWrapper& w, std::string& raw) { +inline std::expected ParseExponent(SourceCodeWrapper& w, std::string& raw) { if (w.Peek() == 'e' || w.Peek() == 'E') { raw.push_back(w.Advance()); @@ -40,7 +40,7 @@ inline bool ParseExponent(SourceCodeWrapper& w, std::string& raw) { } if (!IsDec(w.Peek())) { - throw LexerError("Malformed exponent"); + return std::unexpected(LexerError("Malformed exponent")); } w.ConsumeWhile(raw, IsDec); @@ -50,35 +50,37 @@ inline bool ParseExponent(SourceCodeWrapper& w, std::string& raw) { return false; } -inline double ParseDoubleStrict(const std::string& raw) { +inline std::expected ParseDoubleStrict(const std::string& raw) { try { return std::stod(raw); } catch (...) { - throw LexerError(std::string("Malformed float literal: ") + raw); + return std::unexpected(LexerError(std::string("Malformed float literal: ") + raw)); } } -inline long long ParseDecIntStrict(const std::string& raw) { +inline std::expected ParseDecIntStrict(const std::string& raw) { try { return std::stoll(raw); } catch (...) { - throw LexerError(std::string("Malformed integer literal: ") + raw); + return std::unexpected(LexerError(std::string("Malformed integer literal: ") + raw)); } } -inline void EnsureNoIdentTail(SourceCodeWrapper& w, const char* ctx) { +inline std::expected EnsureNoIdentTail(SourceCodeWrapper& w, const char* ctx) { if (IsIdentStart(w.Peek())) { - throw LexerError(std::string("Unexpected identifier after ") + ctx); + return std::unexpected(LexerError(std::string("Unexpected identifier after ") + ctx)); } + return {}; } -inline void EnsureNoSecondDotWithDigits(SourceCodeWrapper& w) { +inline std::expected EnsureNoSecondDotWithDigits(SourceCodeWrapper& w) { if (w.Peek() == '.' && IsDec(w.Peek(1))) { - throw LexerError("Malformed float literal: duplicate decimal point"); + return std::unexpected(LexerError("Malformed float literal: duplicate decimal point")); } + return {}; } -OptToken NumberHandler::Scan(SourceCodeWrapper& wrapper) { +std::expected NumberHandler::Scan(SourceCodeWrapper& wrapper) { std::string raw; const char first = wrapper.CurrentChar(); @@ -90,12 +92,24 @@ OptToken NumberHandler::Scan(SourceCodeWrapper& wrapper) { } wrapper.ConsumeWhile(raw, IsDec); - ParseExponent(wrapper, raw); - EnsureNoSecondDotWithDigits(wrapper); - EnsureNoIdentTail(wrapper, "number"); - const double v = ParseDoubleStrict(raw); + auto exp_result = ParseExponent(wrapper, raw); + if (!exp_result) { + return std::unexpected(exp_result.error()); + } + auto dot_result = EnsureNoSecondDotWithDigits(wrapper); + if (!dot_result) { + return std::unexpected(dot_result.error()); + } + auto ident_result = EnsureNoIdentTail(wrapper, "number"); + if (!ident_result) { + return std::unexpected(ident_result.error()); + } + auto v_result = ParseDoubleStrict(raw); + if (!v_result) { + return std::unexpected(v_result.error()); + } - return TokenFactory::MakeFloatLiteral(std::move(raw), v, wrapper.GetLine(), wrapper.GetTokenCol()); + return TokenFactory::MakeFloatLiteral(std::move(raw), v_result.value(), wrapper.GetLine(), wrapper.GetTokenCol()); } wrapper.RetreatOne(); @@ -105,16 +119,19 @@ OptToken NumberHandler::Scan(SourceCodeWrapper& wrapper) { raw.push_back(wrapper.Advance()); if (!IsHex(wrapper.Peek())) { - throw LexerError("Malformed hex literal: expected hex digit after 0x"); + return std::unexpected(LexerError("Malformed hex literal: expected hex digit after 0x")); } wrapper.ConsumeWhile(raw, IsHex); if (wrapper.Peek() == '.') { - throw LexerError("Hex literal cannot have decimal point"); + return std::unexpected(LexerError("Hex literal cannot have decimal point")); } - EnsureNoIdentTail(wrapper, "hex literal"); + auto ident_result = EnsureNoIdentTail(wrapper, "hex literal"); + if (!ident_result) { + return std::unexpected(ident_result.error()); + } long long val = 0; @@ -139,16 +156,19 @@ OptToken NumberHandler::Scan(SourceCodeWrapper& wrapper) { raw.push_back(wrapper.Advance()); if (!IsBin(wrapper.Peek())) { - throw LexerError("Malformed binary literal: expected binary digit after 0b"); + return std::unexpected(LexerError("Malformed binary literal: expected binary digit after 0b")); } wrapper.ConsumeWhile(raw, IsBin); if (wrapper.Peek() == '.') { - throw LexerError("Binary literal cannot have decimal point"); + return std::unexpected(LexerError("Binary literal cannot have decimal point")); } - EnsureNoIdentTail(wrapper, "binary literal"); + auto ident_result = EnsureNoIdentTail(wrapper, "binary literal"); + if (!ident_result) { + return std::unexpected(ident_result.error()); + } long long val = 0; for (size_t i = 2; i < raw.size(); ++i) { @@ -167,29 +187,57 @@ OptToken NumberHandler::Scan(SourceCodeWrapper& wrapper) { wrapper.ConsumeWhile(raw, IsDec); } - ParseExponent(wrapper, raw); - EnsureNoSecondDotWithDigits(wrapper); - EnsureNoIdentTail(wrapper, "number"); - const double v = ParseDoubleStrict(raw); - return TokenFactory::MakeFloatLiteral(std::move(raw), v, wrapper.GetLine(), wrapper.GetTokenCol()); + auto exp_result = ParseExponent(wrapper, raw); + if (!exp_result) { + return std::unexpected(exp_result.error()); + } + auto dot_result = EnsureNoSecondDotWithDigits(wrapper); + if (!dot_result) { + return std::unexpected(dot_result.error()); + } + auto ident_result = EnsureNoIdentTail(wrapper, "number"); + if (!ident_result) { + return std::unexpected(ident_result.error()); + } + auto v_result = ParseDoubleStrict(raw); + if (!v_result) { + return std::unexpected(v_result.error()); + } + return TokenFactory::MakeFloatLiteral(std::move(raw), v_result.value(), wrapper.GetLine(), wrapper.GetTokenCol()); } - const bool had_exp = ParseExponent(wrapper, raw); + auto exp_result = ParseExponent(wrapper, raw); + if (!exp_result) { + return std::unexpected(exp_result.error()); + } + const bool had_exp = exp_result.value(); if (had_exp) { - EnsureNoIdentTail(wrapper, "number"); - const double v = ParseDoubleStrict(raw); - return TokenFactory::MakeFloatLiteral(std::move(raw), v, wrapper.GetLine(), wrapper.GetTokenCol()); + auto ident_result = EnsureNoIdentTail(wrapper, "number"); + if (!ident_result) { + return std::unexpected(ident_result.error()); + } + auto v_result = ParseDoubleStrict(raw); + if (!v_result) { + return std::unexpected(v_result.error()); + } + return TokenFactory::MakeFloatLiteral(std::move(raw), v_result.value(), wrapper.GetLine(), wrapper.GetTokenCol()); } if (!raw.empty() && (raw.back() == 'e' || raw.back() == 'E' || raw.back() == '+' || raw.back() == '-')) { - throw LexerError(std::string("Malformed float literal: ") + raw); + return std::unexpected(LexerError(std::string("Malformed float literal: ") + raw)); } - EnsureNoIdentTail(wrapper, "integer literal"); - const long long vi = ParseDecIntStrict(raw); + auto ident_result = EnsureNoIdentTail(wrapper, "integer literal"); + if (!ident_result) { + return std::unexpected(ident_result.error()); + } + auto vi_result = ParseDecIntStrict(raw); + if (!vi_result) { + return std::unexpected(vi_result.error()); + } - return TokenFactory::MakeIntLiteral(std::move(raw), vi, wrapper.GetLine(), wrapper.GetTokenCol()); + return TokenFactory::MakeIntLiteral(std::move(raw), vi_result.value(), wrapper.GetLine(), wrapper.GetTokenCol()); } } // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/NumberHandler.hpp b/lib/lexer/handlers/NumberHandler.hpp index ef33f9a..10cef26 100644 --- a/lib/lexer/handlers/NumberHandler.hpp +++ b/lib/lexer/handlers/NumberHandler.hpp @@ -7,7 +7,7 @@ namespace ovum::compiler::lexer { class NumberHandler : public Handler { public: - OptToken Scan(SourceCodeWrapper& wrapper) override; + std::expected Scan(SourceCodeWrapper& wrapper) override; }; } // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/OperatorHandler.cpp b/lib/lexer/handlers/OperatorHandler.cpp index ae0f67b..793b07a 100644 --- a/lib/lexer/handlers/OperatorHandler.cpp +++ b/lib/lexer/handlers/OperatorHandler.cpp @@ -4,7 +4,7 @@ namespace ovum::compiler::lexer { -OptToken OperatorHandler::Scan(SourceCodeWrapper& wrapper) { +std::expected OperatorHandler::Scan(SourceCodeWrapper& wrapper) { std::string op; op.push_back(wrapper.CurrentChar()); char p = wrapper.Peek(); diff --git a/lib/lexer/handlers/OperatorHandler.hpp b/lib/lexer/handlers/OperatorHandler.hpp index e8168d7..2603c5f 100644 --- a/lib/lexer/handlers/OperatorHandler.hpp +++ b/lib/lexer/handlers/OperatorHandler.hpp @@ -7,7 +7,7 @@ namespace ovum::compiler::lexer { class OperatorHandler : public Handler { public: - OptToken Scan(SourceCodeWrapper& wrapper) override; + std::expected Scan(SourceCodeWrapper& wrapper) override; }; } // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/PunctHandler.cpp b/lib/lexer/handlers/PunctHandler.cpp index 3a3b3ec..ad4356c 100644 --- a/lib/lexer/handlers/PunctHandler.cpp +++ b/lib/lexer/handlers/PunctHandler.cpp @@ -4,7 +4,7 @@ namespace ovum::compiler::lexer { -OptToken PunctHandler::Scan(SourceCodeWrapper& wrapper) { +std::expected PunctHandler::Scan(SourceCodeWrapper& wrapper) { char c = wrapper.CurrentChar(); return std::make_optional(TokenFactory::MakePunct(c, wrapper.GetLine(), wrapper.GetTokenCol())); } diff --git a/lib/lexer/handlers/PunctHandler.hpp b/lib/lexer/handlers/PunctHandler.hpp index bbc6296..baf4bc8 100644 --- a/lib/lexer/handlers/PunctHandler.hpp +++ b/lib/lexer/handlers/PunctHandler.hpp @@ -7,7 +7,7 @@ namespace ovum::compiler::lexer { class PunctHandler : public Handler { public: - OptToken Scan(SourceCodeWrapper& wrapper) override; + std::expected Scan(SourceCodeWrapper& wrapper) override; }; } // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/SlashHandler.cpp b/lib/lexer/handlers/SlashHandler.cpp index 76b2796..9637ee0 100644 --- a/lib/lexer/handlers/SlashHandler.cpp +++ b/lib/lexer/handlers/SlashHandler.cpp @@ -5,7 +5,7 @@ namespace ovum::compiler::lexer { -OptToken SlashHandler::Scan(SourceCodeWrapper& wrapper) { +std::expected SlashHandler::Scan(SourceCodeWrapper& wrapper) { if (wrapper.Peek() == '/') { std::string txt; @@ -40,7 +40,7 @@ OptToken SlashHandler::Scan(SourceCodeWrapper& wrapper) { } if (!closed) { - throw LexerError("Unterminated block comment"); + return std::unexpected(LexerError("Unterminated block comment")); } if (wrapper.IsKeepComments()) { diff --git a/lib/lexer/handlers/SlashHandler.hpp b/lib/lexer/handlers/SlashHandler.hpp index 5329841..5719c95 100644 --- a/lib/lexer/handlers/SlashHandler.hpp +++ b/lib/lexer/handlers/SlashHandler.hpp @@ -7,7 +7,7 @@ namespace ovum::compiler::lexer { class SlashHandler : public Handler { public: - OptToken Scan(SourceCodeWrapper& wrapper) override; + std::expected Scan(SourceCodeWrapper& wrapper) override; }; } // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/StringHandler.cpp b/lib/lexer/handlers/StringHandler.cpp index ead4537..c7f2d8f 100644 --- a/lib/lexer/handlers/StringHandler.cpp +++ b/lib/lexer/handlers/StringHandler.cpp @@ -7,7 +7,7 @@ namespace ovum::compiler::lexer { -OptToken StringHandler::Scan(SourceCodeWrapper& wrapper) { +std::expected StringHandler::Scan(SourceCodeWrapper& wrapper) { std::string raw; std::string out; raw.push_back('"'); @@ -23,7 +23,7 @@ OptToken StringHandler::Scan(SourceCodeWrapper& wrapper) { if (c == '\\') { if (wrapper.IsAtEnd()) { - throw LexerError("Unterminated string literal (backslash at EOF)"); + return std::unexpected(LexerError("Unterminated string literal (backslash at EOF)")); } char e = wrapper.Advance(); @@ -49,18 +49,18 @@ OptToken StringHandler::Scan(SourceCodeWrapper& wrapper) { out.push_back('\0'); break; default: - throw LexerError(std::string("Unknown escape in string literal: \\") + e); + return std::unexpected(LexerError(std::string("Unknown escape in string literal: \\") + e)); } } else { if (c == '\n') { - throw LexerError("Unterminated string literal (newline inside)"); + return std::unexpected(LexerError("Unterminated string literal (newline inside)")); } out.push_back(c); } } - throw LexerError("Unterminated string literal (EOF reached)"); + return std::unexpected(LexerError("Unterminated string literal (EOF reached)")); } } // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/StringHandler.hpp b/lib/lexer/handlers/StringHandler.hpp index 78720f9..929c8d4 100644 --- a/lib/lexer/handlers/StringHandler.hpp +++ b/lib/lexer/handlers/StringHandler.hpp @@ -7,7 +7,7 @@ namespace ovum::compiler::lexer { class StringHandler : public Handler { public: - OptToken Scan(SourceCodeWrapper& wrapper) override; + std::expected Scan(SourceCodeWrapper& wrapper) override; }; } // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/WhitespaceHandler.cpp b/lib/lexer/handlers/WhitespaceHandler.cpp index 694eb74..060cb0b 100644 --- a/lib/lexer/handlers/WhitespaceHandler.cpp +++ b/lib/lexer/handlers/WhitespaceHandler.cpp @@ -2,7 +2,7 @@ namespace ovum::compiler::lexer { -OptToken WhitespaceHandler::Scan(SourceCodeWrapper& wrapper) { +std::expected WhitespaceHandler::Scan(SourceCodeWrapper& wrapper) { while (!wrapper.IsAtEnd()) { if (char p = wrapper.Peek(); p != ' ' && p != '\t' && p != '\r') { break; diff --git a/lib/lexer/handlers/WhitespaceHandler.hpp b/lib/lexer/handlers/WhitespaceHandler.hpp index e3bf5d4..f9f7a29 100644 --- a/lib/lexer/handlers/WhitespaceHandler.hpp +++ b/lib/lexer/handlers/WhitespaceHandler.hpp @@ -7,7 +7,7 @@ namespace ovum::compiler::lexer { class WhitespaceHandler : public Handler { public: - OptToken Scan(SourceCodeWrapper& wrapper) override; + std::expected Scan(SourceCodeWrapper& wrapper) override; }; } // namespace ovum::compiler::lexer diff --git a/lib/preprocessor/Preprocessor.cpp b/lib/preprocessor/Preprocessor.cpp index adecb8d..6ff0857 100644 --- a/lib/preprocessor/Preprocessor.cpp +++ b/lib/preprocessor/Preprocessor.cpp @@ -1,6 +1,5 @@ #include "Preprocessor.hpp" -#include #include #include @@ -33,14 +32,14 @@ std::expected, PreprocessorError> Preprocessor::Process() } lexer::Lexer lexer(content, false); - std::vector tokens; + auto tokens_result = lexer.Tokenize(); - try { - tokens = std::move(lexer.Tokenize()); - } catch (const std::exception& e) { - return std::unexpected(PreprocessorError(e.what())); + if (!tokens_result) { + return std::unexpected(PreprocessorError(tokens_result.error().what())); } + std::vector tokens = std::move(tokens_result.value()); + for (std::unique_ptr& processor : token_processors_) { std::expected, PreprocessorError> processor_result = std::move(processor->Process(tokens)); diff --git a/lib/preprocessor/import_processor/TokenImportProcessor.cpp b/lib/preprocessor/import_processor/TokenImportProcessor.cpp index 688068d..660b9b5 100644 --- a/lib/preprocessor/import_processor/TokenImportProcessor.cpp +++ b/lib/preprocessor/import_processor/TokenImportProcessor.cpp @@ -125,14 +125,15 @@ std::expected TokenImportProcessor::GatherDependencies( std::string_view content_view(content_str); lexer::Lexer lexer(content_view, false); - std::vector raw_tokens; + auto raw_tokens_result = lexer.Tokenize(); - try { - raw_tokens = lexer.Tokenize(); - } catch (const std::exception& e) { - return std::unexpected(PreprocessorError("Lexer error for " + dep_path.string() + ": " + e.what())); + if (!raw_tokens_result) { + return std::unexpected( + PreprocessorError("Lexer error for " + dep_path.string() + ": " + raw_tokens_result.error().what())); } + std::vector raw_tokens = std::move(raw_tokens_result.value()); + std::expected sub_result = GatherDependencies(dep_path, raw_tokens); if (!sub_result) { diff --git a/tests/lexer_big_programs_tests.cpp b/tests/lexer_big_programs_tests.cpp index 67c5749..a291fa1 100644 --- a/tests/lexer_big_programs_tests.cpp +++ b/tests/lexer_big_programs_tests.cpp @@ -17,8 +17,9 @@ TEST(LexerUnitTestSuite, ExampleFundamentals) { sys::Print(i.ToString()) })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun","ExampleFundamentals","(",")",":","Void","{","\\n", @@ -55,8 +56,9 @@ val count: Int = 0 val pi: Float = 3.14 })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun","ExampleReferences","(",")",":","Void","{","\\n", @@ -92,8 +94,9 @@ val optArr: IntArray? = null val safeInt: Int? = 42 })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "ExampleNullable", "(", ")", ":", "Void", "{", "\\n", @@ -120,8 +123,9 @@ intArr[1] := 2 val emptyList: IntArray = IntArray(0) })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun","ExampleArrays","(",")",":","Void","{","\\n", @@ -155,8 +159,9 @@ val shape: IShape = Circle(5.0) val circle: Circle? = Circle(10.0) })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "interface", "IShape", "{", "fun", "Area", "(", ")", ":", @@ -197,8 +202,9 @@ val obj: ExampleClass = ExampleClass() obj.mutable = 30.0 })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "class","ExampleClass","{","\\n", @@ -227,8 +233,9 @@ else if (x < 0) { sys::Print("Negative") } else { sys::Print("Zero") } })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun","SimpleIf","(","x",":","int",")",":","Void","{","\\n", @@ -256,8 +263,9 @@ else if (!a ^ b) { sys::Print("Exactly one false") } else { sys::Print("Both false") } })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun","BooleanConditions","(","a",":","bool",",","b",":","bool",")",":","Void","{","\\n", @@ -289,8 +297,9 @@ else if (obj is Int) { val num: Int = obj as Int sys::Print(num.ToString()) } else { sys::Print("Unknown type") } })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "TypeConditions", "(", "obj", ":", "Object", ")", ":", "Void", "{", @@ -328,8 +337,9 @@ if (value > 10) { return value * 2 } else { return value } })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "ElvisInIf", "(", "opt", ":", "Int", "?", ")", ":", "int", "{", "\\n", "val", "value", @@ -352,8 +362,9 @@ else if (x > 10) return "Big" else return "Other" })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "WhenLike", "(", "x", ":", "int", ")", ":", "String", "{", "\\n", "if", @@ -383,8 +394,9 @@ while (counter < n) { return counter })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "WhileExample", "(", "n", ":", "int", ")", ":", "int", "{", @@ -425,8 +437,9 @@ for (num in arr) { if (num < 0) break sum = sum + num } return sum })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "ForExample", "(", "arr", ":", "IntArray", ")", ":", "int", "{", "\\n", @@ -455,8 +468,9 @@ if (i > 5) break } } })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "NestedLoops", "(", "matrix", ":", "IntArrayArray", ")", ":", "Void", "{", @@ -503,8 +517,9 @@ while (i < arr.Length()) { sys::Print((i.ToString() + ": " + arr[i].ToString())) i = i + 1 }})OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "ForWithIndex", "(", "arr", ":", "IntArray", ")", ":", "Void", "{", @@ -543,8 +558,9 @@ fromLiteral[0] := 1 fromLiteral[1] := 2 })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "ListCreation", "(", ")", ":", "Void", "{", @@ -592,8 +608,9 @@ if (list[idx] % 2 == 0) { list[idx] := list[idx] * 2 } idx = idx + 1 } return total })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "ListIteration", "(", "list", ":", "IntArray", ")", ":", "int", "{", @@ -635,8 +652,9 @@ newList[original.Length()] := 100 return newList })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "ListOps", "(", "original", ":", "IntArray", ")", ":", "IntArray", "{", "\\n", @@ -667,8 +685,9 @@ for (num in list) { if (num == target) { return Int(num) } } return null })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "FindInList", "(", "list", ":", "IntArray", ",", "target", ":", "int", ")", ":", "Int", "?", "{", @@ -696,8 +715,9 @@ val length: int = obj?.Length() ?: 0 sys::Print(length.ToString()) })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "SafeCall", "(", "obj", ":", "String", "?", ")", ":", "Void", "{", "\\n", "val", @@ -719,8 +739,9 @@ val str: String = optInt?.ToString() ?: "Unknown" return default })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "ElvisExamples", "(", "optInt", ":", "Int", "?", ")", ":", "int", "{", @@ -750,8 +771,9 @@ val safe: Object? = opt if (safe == null) { sys::Print("Is null") } })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "NullChecks", "(", "opt", ":", "Object", "?", ")", ":", @@ -783,8 +805,9 @@ sys::Print(value.ToString()) } val risky: Int = obj as Int })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "CastNullable", "(", "obj", ":", "Object", "?", ")", ":", "Void", "{", @@ -822,8 +845,9 @@ while (i < src.Length()) { copy[i] := src[i] i = i + 1 } return copy })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "CopyNullable", "(", "src", ":", "IntArray", "?", ")", ":", "IntArray", "?", "{", @@ -864,8 +888,9 @@ val len: int = nested?.Inner?.Length() ?: 0 sys::Print(len.ToString()) })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "class", "Nested", "{", @@ -903,8 +928,9 @@ val bVal: int = b ?: 0 return Int(aVal + bVal) })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "pure", "fun", "SafeAdd", "(", "a", ":", "Int", "?", ",", "b", ":", "Int", "?", ")", ":", @@ -927,8 +953,9 @@ if (ptr != null) { val bytes: ByteArray = (ptr as ByteArray) } }})OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "UnsafeNullable", "(", "ptr", ":", "Pointer", "?", ")", ":", "Void", "{", diff --git a/tests/lexer_tests.cpp b/tests/lexer_tests.cpp index 26a88aa..b158d2f 100644 --- a/tests/lexer_tests.cpp +++ b/tests/lexer_tests.cpp @@ -9,8 +9,9 @@ using ovum::compiler::lexer::Lexer; TEST(LexerUnitTestSuite, EmptyString) { const std::string src = ""; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {}; std::vector expected_types = {}; LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -19,8 +20,9 @@ TEST(LexerUnitTestSuite, EmptyString) { TEST(LexerUnitTestSuite, SingleCharacter) { const std::string src = "c"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"c"}; std::vector expected_types = {"IDENT"}; LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -31,8 +33,9 @@ TEST(LexerUnitTestSuite, Keywords) { "fun pure val var class interface implements override if else while for in return break continue unsafe is as " "typealias"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = { "fun", "pure", "val", "var", "class", "interface", "implements", "override", "if", "else", "while", "for", "in", "return", "break", "continue", "unsafe", "is", "as", "typealias"}; @@ -43,8 +46,9 @@ TEST(LexerUnitTestSuite, Keywords) { TEST(LexerUnitTestSuite, Punctuation) { const std::string src = "{ } ( ) [ ] , : ;"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"{", "}", "(", ")", "[", "]", ",", ":", ";"}; std::vector expected_types(expected_lexemes.size(), "PUNCT"); LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -53,8 +57,9 @@ TEST(LexerUnitTestSuite, Punctuation) { TEST(LexerUnitTestSuite, Identifiers) { const std::string src = "abc ABC _abc ovum ExampleFundamentals Void String IntArray"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = { "abc", "ABC", "_abc", "ovum", "ExampleFundamentals", "Void", "String", "IntArray"}; std::vector expected_types(expected_lexemes.size(), "IDENT"); @@ -64,8 +69,9 @@ TEST(LexerUnitTestSuite, Identifiers) { TEST(LexerUnitTestSuite, OperatorsSimple) { const std::string src = "+ - * / % < > ! . ?"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"+", "-", "*", "/", "%", "<", ">", "!", ".", "?"}; std::vector expected_types(expected_lexemes.size(), "OPERATOR"); LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -74,8 +80,9 @@ TEST(LexerUnitTestSuite, OperatorsSimple) { TEST(LexerUnitTestSuite, OperatorsMultiChar) { const std::string src = "== != <= >= && || = := ::"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"==", "!=", "<=", ">=", "&&", "||", "=", ":=", "::"}; std::vector expected_types(expected_lexemes.size(), "OPERATOR"); LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -84,8 +91,9 @@ TEST(LexerUnitTestSuite, OperatorsMultiChar) { TEST(LexerUnitTestSuite, OperatorsNullSafe) { const std::string src = "?. ?:"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"?.", "?:"}; std::vector expected_types(expected_lexemes.size(), "OPERATOR"); LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -94,8 +102,9 @@ TEST(LexerUnitTestSuite, OperatorsNullSafe) { TEST(LexerUnitTestSuite, OperatorXor) { const std::string src = "^"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"^"}; std::vector expected_types = {"OPERATOR"}; LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -104,8 +113,9 @@ TEST(LexerUnitTestSuite, OperatorXor) { TEST(LexerUnitTestSuite, LiteralsInt) { const std::string src = "0 42 255"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"0", "42", "255"}; std::vector expected_types(expected_lexemes.size(), "LITERAL:Int"); LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -114,8 +124,9 @@ TEST(LexerUnitTestSuite, LiteralsInt) { TEST(LexerUnitTestSuite, LiteralsFloat) { const std::string src = "3.14 1e3 .5 5. 2.0E-2"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"3.14", "1e3", ".5", "5.", "2.0E-2"}; std::vector expected_types(expected_lexemes.size(), "LITERAL:Float"); LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -124,8 +135,9 @@ TEST(LexerUnitTestSuite, LiteralsFloat) { TEST(LexerUnitTestSuite, LiteralsChar) { const std::string src = "'A' '\\n'"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"'A'", "'\\n'"}; std::vector expected_types(expected_lexemes.size(), "LITERAL:Char"); LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -134,8 +146,9 @@ TEST(LexerUnitTestSuite, LiteralsChar) { TEST(LexerUnitTestSuite, LiteralsBool) { const std::string src = "true false"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"true", "false"}; std::vector expected_types(expected_lexemes.size(), "LITERAL:Bool"); LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -144,8 +157,9 @@ TEST(LexerUnitTestSuite, LiteralsBool) { TEST(LexerUnitTestSuite, KeywordNull) { const std::string src = "null"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"null"}; std::vector expected_types = {"KEYWORD"}; LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -154,8 +168,9 @@ TEST(LexerUnitTestSuite, KeywordNull) { TEST(LexerUnitTestSuite, LiteralsString) { const std::string src = R"("hello" "world\n" "")"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {R"("hello")", R"("world\n")", R"("")"}; std::vector expected_types(expected_lexemes.size(), "LITERAL:String"); LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -164,8 +179,9 @@ TEST(LexerUnitTestSuite, LiteralsString) { TEST(LexerUnitTestSuite, Preprocessor) { const std::string src = "#import #define #ifdef"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"#import", "#define", "#ifdef"}; std::vector expected_types(expected_lexemes.size(), "KEYWORD"); LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -174,8 +190,9 @@ TEST(LexerUnitTestSuite, Preprocessor) { TEST(LexerUnitTestSuite, Newline) { const std::string src = "\n"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"\\n"}; std::vector expected_types = {"NEWLINE"}; LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -184,8 +201,9 @@ TEST(LexerUnitTestSuite, Newline) { TEST(LexerUnitTestSuite, LiteralsIntHex) { const std::string src = "0x1A 0xFF"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"0x1A", "0xFF"}; std::vector expected_types(expected_lexemes.size(), "LITERAL:Int"); LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -194,8 +212,9 @@ TEST(LexerUnitTestSuite, LiteralsIntHex) { TEST(LexerUnitTestSuite, LiteralsIntBinary) { const std::string src = "0b1010 0b1111"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"0b1010", "0b1111"}; std::vector expected_types(expected_lexemes.size(), "LITERAL:Int"); LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -204,8 +223,9 @@ TEST(LexerUnitTestSuite, LiteralsIntBinary) { TEST(LexerUnitTestSuite, LiteralsIntNegative) { const std::string src = "-17 -42"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"-", "17", "-", "42"}; std::vector expected_types = {"OPERATOR", "LITERAL:Int", "OPERATOR", "LITERAL:Int"}; LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -214,8 +234,9 @@ TEST(LexerUnitTestSuite, LiteralsIntNegative) { TEST(LexerUnitTestSuite, LiteralsFloatSpecial) { const std::string src = "Infinity -Infinity NaN"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"Infinity", "-", "Infinity", "NaN"}; std::vector expected_types = {"LITERAL:Float", "OPERATOR", "LITERAL:Float", "LITERAL:Float"}; LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -224,8 +245,9 @@ TEST(LexerUnitTestSuite, LiteralsFloatSpecial) { TEST(LexerUnitTestSuite, LiteralsCharEscapes) { const std::string src = R"('A' '\n' '\t' '\0' '\'')"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"'A'", "'\\n'", "'\\t'", "'\\0'", "'\\''"}; std::vector expected_types(expected_lexemes.size(), "LITERAL:Char"); LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -234,8 +256,9 @@ TEST(LexerUnitTestSuite, LiteralsCharEscapes) { TEST(LexerUnitTestSuite, WhitespaceSkipping) { const std::string src = "fun val \t if"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"fun", "val", "if"}; std::vector expected_types(expected_lexemes.size(), "KEYWORD"); LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -245,8 +268,9 @@ TEST(LexerUnitTestSuite, SingleLineComment) { const std::string src = R"OVUM(// This is a single-line comment fun Main)OVUM"; Lexer lexer(src, true); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {" This is a single-line comment", "\\n", "fun", "Main"}; std::vector expected_types = {"COMMENT", "NEWLINE", "KEYWORD", "IDENT"}; LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -256,8 +280,9 @@ TEST(LexerUnitTestSuite, SingleLineCommentNoSpace) { const std::string src = R"OVUM(//This is a single-line comment without space )OVUM"; Lexer lexer(src, true); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"This is a single-line comment without space", "\\n"}; std::vector expected_types = {"COMMENT", "NEWLINE"}; LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -267,8 +292,9 @@ TEST(LexerUnitTestSuite, MultiLineCommentPositive) { const std::string src = R"OVUM(/* This is a properly closed multi-line comment */)OVUM"; Lexer lexer(src, true); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {" This is a properly closed\nmulti-line comment "}; std::vector expected_types = {"COMMENT"}; LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -278,8 +304,9 @@ TEST(LexerUnitTestSuite, MultiLineCommentWithSlashesAndStars) { const std::string src = R"OVUM(/* This comment contains / and * signs: /* // */ fun Main())OVUM"; Lexer lexer(src, true); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = { " This comment contains / and * signs: /* // ", "\\n", "fun", "Main", "(", ")"}; std::vector expected_types = {"COMMENT", "NEWLINE", "KEYWORD", "IDENT", "PUNCT", "PUNCT"}; @@ -289,8 +316,9 @@ fun Main())OVUM"; TEST(LexerUnitTestSuite, IncorrectCommentWithSlashesAndStars) { const std::string src = R"OVUM(/* This comment contains / and * signs: /* // */ */)OVUM"; Lexer lexer(src, true); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {" This comment contains / and * signs: /* // ", "*", "/"}; std::vector expected_types = {"COMMENT", "OPERATOR", "OPERATOR"}; LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -303,8 +331,9 @@ Third line after empty line Fifth line */)OVUM"; Lexer lexer(src, true); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {" First line\n\nThird line after empty line\n\nFifth line "}; std::vector expected_types = {"COMMENT"}; LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -315,91 +344,106 @@ Fifth line */)OVUM"; TEST(LexerUnitTestSuite, InvalidCharacter) { const std::string src = "fun @main(): Void {}"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error for invalid character"; } TEST(LexerUnitTestSuite, InvalidCharacterInIdentifier) { const std::string src = "val name$var: int = 0"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } TEST(LexerUnitTestSuite, InvalidIdentifierStartingWithDigit) { const std::string src = "val 1abc: int = 0"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } TEST(LexerUnitTestSuite, InvalidHexNumber) { const std::string src = "val x: int = 0xG"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } TEST(LexerUnitTestSuite, InvalidBinaryNumber) { const std::string src = "val x: int = 0b2"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } TEST(LexerUnitTestSuite, InvalidFloatNumber) { const std::string src = "val x: float = 1.2.3"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } TEST(LexerUnitTestSuite, InvalidFloatIncompleteExp) { const std::string src = "val x: float = 1e"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } TEST(LexerUnitTestSuite, UnterminatedString) { const std::string src = R"(val msg: String = "hello)"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } TEST(LexerUnitTestSuite, InvalidEscapeInString) { const std::string src = R"(val msg: String = "\z")"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } TEST(LexerUnitTestSuite, UnterminatedChar) { const std::string src = "val c: char = 'A"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } TEST(LexerUnitTestSuite, EmptyCharLiteral) { const std::string src = R"(val c: char = '')"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } TEST(LexerUnitTestSuite, MultiCharLiteral) { const std::string src = "val c: char = 'AB'"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } TEST(LexerUnitTestSuite, InvalidEscapeInChar) { const std::string src = "val c: char = '\\z'"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } TEST(LexerUnitTestSuite, InvalidPreprocessor) { const std::string src = "#invalid"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } TEST(LexerUnitTestSuite, MultipleErrorsInOneFile) { const std::string src = R"(fun main@(): Void { val x = "unclosed; val y = 0xG })"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } TEST(LexerUnitTestSuite, MultiLineCommentNegativeUnclosed) { @@ -408,5 +452,6 @@ fun Main(): Void { val x: int = 42 })OVUM"; Lexer lexer(src, true); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } diff --git a/tests/test_suites/PreprocessorUnitTestSuite.cpp b/tests/test_suites/PreprocessorUnitTestSuite.cpp index cda1bb2..df105e6 100644 --- a/tests/test_suites/PreprocessorUnitTestSuite.cpp +++ b/tests/test_suites/PreprocessorUnitTestSuite.cpp @@ -79,7 +79,11 @@ std::expected, std::string> PreprocessorUnitTestSuite::Tok ovum::compiler::lexer::Lexer lexer(content, false); auto tokens_result = lexer.Tokenize(); - return tokens_result; + if (!tokens_result) { + return std::unexpected("Lexer error: " + std::string(tokens_result.error().what())); + } + + return tokens_result.value(); } bool PreprocessorUnitTestSuite::CompareTokenSequences(const std::vector& actual, From 0e3dacf4117f4d6d6abbd9847dc68e1f17f816b8 Mon Sep 17 00:00:00 2001 From: bialger Date: Sat, 8 Nov 2025 22:17:13 +0300 Subject: [PATCH 3/3] build: integrate ovumcommon library and remove tokens library from project structure --- cmake/IncludeExternalLibraries.cmake | 9 +++ lib/CMakeLists.txt | 1 - lib/lexer/Lexer.cpp | 4 +- lib/lexer/Lexer.hpp | 3 +- lib/lexer/handlers/CharHandler.cpp | 3 +- lib/lexer/handlers/ColonHandler.cpp | 3 +- lib/lexer/handlers/Handler.hpp | 3 +- lib/lexer/handlers/IdentifierHandler.cpp | 3 +- lib/lexer/handlers/NewlineHandler.cpp | 2 +- lib/lexer/handlers/NumberHandler.cpp | 3 +- lib/lexer/handlers/OperatorHandler.cpp | 2 +- lib/lexer/handlers/PunctHandler.cpp | 2 +- lib/lexer/handlers/SlashHandler.cpp | 3 +- lib/lexer/handlers/StringHandler.cpp | 3 +- lib/preprocessor/TokenProcessor.hpp | 3 +- .../TokenDirectivesProcessor.hpp | 3 +- .../handlers/DefaultHandler.hpp | 3 +- .../handlers/DefineHandler.hpp | 3 +- .../handlers/DirectiveHandler.hpp | 3 +- .../handlers/ElseHandler.hpp | 3 +- .../handlers/EndifHandler.hpp | 3 +- .../handlers/IfdefHandler.hpp | 3 +- .../handlers/IfndefHandler.hpp | 3 +- .../handlers/UndefHandler.hpp | 3 +- .../import_processor/TokenImportProcessor.hpp | 3 +- lib/tokens/CMakeLists.txt | 28 ------- lib/tokens/CommentToken.cpp | 37 --------- lib/tokens/CommentToken.hpp | 34 --------- lib/tokens/EofToken.cpp | 36 --------- lib/tokens/EofToken.hpp | 33 -------- lib/tokens/IdentToken.cpp | 40 ---------- lib/tokens/IdentToken.hpp | 36 --------- lib/tokens/KeywordToken.cpp | 37 --------- lib/tokens/KeywordToken.hpp | 34 --------- lib/tokens/LiteralToken.cpp | 53 ------------- lib/tokens/LiteralToken.hpp | 41 ---------- lib/tokens/NewlineToken.cpp | 36 --------- lib/tokens/NewlineToken.hpp | 34 --------- lib/tokens/OperatorToken.cpp | 37 --------- lib/tokens/OperatorToken.hpp | 35 --------- lib/tokens/PunctToken.cpp | 39 ---------- lib/tokens/PunctToken.hpp | 36 --------- lib/tokens/Token.hpp | 33 -------- lib/tokens/TokenFactory.cpp | 76 ------------------- lib/tokens/TokenFactory.hpp | 38 ---------- lib/tokens/TokenPosition.cpp | 16 ---- lib/tokens/TokenPosition.hpp | 23 ------ lib/tokens/TokenVisitor.hpp | 38 ---------- lib/tokens/values/BoolValue.cpp | 20 ----- lib/tokens/values/BoolValue.hpp | 27 ------- lib/tokens/values/CharValue.cpp | 20 ----- lib/tokens/values/CharValue.hpp | 27 ------- lib/tokens/values/FloatValue.cpp | 24 ------ lib/tokens/values/FloatValue.hpp | 27 ------- lib/tokens/values/IntValue.cpp | 20 ----- lib/tokens/values/IntValue.hpp | 28 ------- lib/tokens/values/StringValue.cpp | 22 ------ lib/tokens/values/StringValue.hpp | 27 ------- lib/tokens/values/Value.hpp | 22 ------ tests/test_suites/LexerUnitTestSuite.hpp | 3 +- .../test_suites/PreprocessorUnitTestSuite.hpp | 3 +- 61 files changed, 56 insertions(+), 1141 deletions(-) delete mode 100644 lib/tokens/CMakeLists.txt delete mode 100644 lib/tokens/CommentToken.cpp delete mode 100644 lib/tokens/CommentToken.hpp delete mode 100644 lib/tokens/EofToken.cpp delete mode 100644 lib/tokens/EofToken.hpp delete mode 100644 lib/tokens/IdentToken.cpp delete mode 100644 lib/tokens/IdentToken.hpp delete mode 100644 lib/tokens/KeywordToken.cpp delete mode 100644 lib/tokens/KeywordToken.hpp delete mode 100644 lib/tokens/LiteralToken.cpp delete mode 100644 lib/tokens/LiteralToken.hpp delete mode 100644 lib/tokens/NewlineToken.cpp delete mode 100644 lib/tokens/NewlineToken.hpp delete mode 100644 lib/tokens/OperatorToken.cpp delete mode 100644 lib/tokens/OperatorToken.hpp delete mode 100644 lib/tokens/PunctToken.cpp delete mode 100644 lib/tokens/PunctToken.hpp delete mode 100644 lib/tokens/Token.hpp delete mode 100644 lib/tokens/TokenFactory.cpp delete mode 100644 lib/tokens/TokenFactory.hpp delete mode 100644 lib/tokens/TokenPosition.cpp delete mode 100644 lib/tokens/TokenPosition.hpp delete mode 100644 lib/tokens/TokenVisitor.hpp delete mode 100644 lib/tokens/values/BoolValue.cpp delete mode 100644 lib/tokens/values/BoolValue.hpp delete mode 100644 lib/tokens/values/CharValue.cpp delete mode 100644 lib/tokens/values/CharValue.hpp delete mode 100644 lib/tokens/values/FloatValue.cpp delete mode 100644 lib/tokens/values/FloatValue.hpp delete mode 100644 lib/tokens/values/IntValue.cpp delete mode 100644 lib/tokens/values/IntValue.hpp delete mode 100644 lib/tokens/values/StringValue.cpp delete mode 100644 lib/tokens/values/StringValue.hpp delete mode 100644 lib/tokens/values/Value.hpp diff --git a/cmake/IncludeExternalLibraries.cmake b/cmake/IncludeExternalLibraries.cmake index d002ab7..95e121b 100644 --- a/cmake/IncludeExternalLibraries.cmake +++ b/cmake/IncludeExternalLibraries.cmake @@ -11,3 +11,12 @@ FetchContent_Declare( ) FetchContent_MakeAvailable(googletest) + +# Ovum Common +FetchContent_Declare( + ovumcommon + GIT_REPOSITORY https://github.com/Ovum-Programming-Language/OvumCommon.git + GIT_TAG main +) + +FetchContent_MakeAvailable(ovumcommon) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index e6238d1..bd0fc3a 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -1,4 +1,3 @@ -add_subdirectory(tokens) add_subdirectory(lexer) add_subdirectory(preprocessor) add_subdirectory(compiler_ui) diff --git a/lib/lexer/Lexer.cpp b/lib/lexer/Lexer.cpp index 442f63c..da817ac 100644 --- a/lib/lexer/Lexer.cpp +++ b/lib/lexer/Lexer.cpp @@ -3,6 +3,8 @@ #include #include +#include + #include "handlers/CharHandler.hpp" #include "handlers/ColonHandler.hpp" #include "handlers/DefaultHandler.hpp" @@ -16,8 +18,6 @@ #include "handlers/StringHandler.hpp" #include "handlers/WhitespaceHandler.hpp" -#include "lib/tokens/TokenFactory.hpp" - namespace ovum::compiler::lexer { namespace { diff --git a/lib/lexer/Lexer.hpp b/lib/lexer/Lexer.hpp index 7e664ef..9eaced0 100644 --- a/lib/lexer/Lexer.hpp +++ b/lib/lexer/Lexer.hpp @@ -8,10 +8,11 @@ #include #include +#include + #include "LexerError.hpp" #include "SourceCodeWrapper.hpp" #include "handlers/Handler.hpp" -#include "lib/tokens/Token.hpp" namespace ovum::compiler::lexer { diff --git a/lib/lexer/handlers/CharHandler.cpp b/lib/lexer/handlers/CharHandler.cpp index f2846f5..3e8b2fe 100644 --- a/lib/lexer/handlers/CharHandler.cpp +++ b/lib/lexer/handlers/CharHandler.cpp @@ -1,7 +1,8 @@ #include "CharHandler.hpp" +#include + #include "lib/lexer/LexerError.hpp" -#include "lib/tokens/TokenFactory.hpp" namespace ovum::compiler::lexer { diff --git a/lib/lexer/handlers/ColonHandler.cpp b/lib/lexer/handlers/ColonHandler.cpp index 77dc49f..fec0f3c 100644 --- a/lib/lexer/handlers/ColonHandler.cpp +++ b/lib/lexer/handlers/ColonHandler.cpp @@ -1,5 +1,6 @@ #include "ColonHandler.hpp" -#include "lib/tokens/TokenFactory.hpp" + +#include namespace ovum::compiler::lexer { diff --git a/lib/lexer/handlers/Handler.hpp b/lib/lexer/handlers/Handler.hpp index f4475eb..6638e64 100644 --- a/lib/lexer/handlers/Handler.hpp +++ b/lib/lexer/handlers/Handler.hpp @@ -4,9 +4,10 @@ #include #include +#include + #include "lib/lexer/LexerError.hpp" #include "lib/lexer/SourceCodeWrapper.hpp" -#include "lib/tokens/Token.hpp" namespace ovum::compiler::lexer { diff --git a/lib/lexer/handlers/IdentifierHandler.cpp b/lib/lexer/handlers/IdentifierHandler.cpp index 98da3c9..23e4afe 100644 --- a/lib/lexer/handlers/IdentifierHandler.cpp +++ b/lib/lexer/handlers/IdentifierHandler.cpp @@ -3,8 +3,9 @@ #include #include +#include + #include "LexerError.hpp" -#include "lib/tokens/TokenFactory.hpp" namespace ovum::compiler::lexer { diff --git a/lib/lexer/handlers/NewlineHandler.cpp b/lib/lexer/handlers/NewlineHandler.cpp index c537793..9def50e 100644 --- a/lib/lexer/handlers/NewlineHandler.cpp +++ b/lib/lexer/handlers/NewlineHandler.cpp @@ -1,6 +1,6 @@ #include "NewlineHandler.hpp" -#include "lib/tokens/TokenFactory.hpp" +#include namespace ovum::compiler::lexer { diff --git a/lib/lexer/handlers/NumberHandler.cpp b/lib/lexer/handlers/NumberHandler.cpp index 373b6ac..9d59c6e 100644 --- a/lib/lexer/handlers/NumberHandler.cpp +++ b/lib/lexer/handlers/NumberHandler.cpp @@ -3,8 +3,9 @@ #include #include +#include + #include "lib/lexer/LexerError.hpp" -#include "lib/tokens/TokenFactory.hpp" namespace ovum::compiler::lexer { diff --git a/lib/lexer/handlers/OperatorHandler.cpp b/lib/lexer/handlers/OperatorHandler.cpp index 793b07a..82c9554 100644 --- a/lib/lexer/handlers/OperatorHandler.cpp +++ b/lib/lexer/handlers/OperatorHandler.cpp @@ -1,6 +1,6 @@ #include "OperatorHandler.hpp" -#include "lib/tokens/TokenFactory.hpp" +#include namespace ovum::compiler::lexer { diff --git a/lib/lexer/handlers/PunctHandler.cpp b/lib/lexer/handlers/PunctHandler.cpp index ad4356c..09eefea 100644 --- a/lib/lexer/handlers/PunctHandler.cpp +++ b/lib/lexer/handlers/PunctHandler.cpp @@ -1,6 +1,6 @@ #include "PunctHandler.hpp" -#include "lib/tokens/TokenFactory.hpp" +#include namespace ovum::compiler::lexer { diff --git a/lib/lexer/handlers/SlashHandler.cpp b/lib/lexer/handlers/SlashHandler.cpp index 9637ee0..6fecd1d 100644 --- a/lib/lexer/handlers/SlashHandler.cpp +++ b/lib/lexer/handlers/SlashHandler.cpp @@ -1,7 +1,8 @@ #include "SlashHandler.hpp" +#include + #include "lib/lexer/LexerError.hpp" -#include "lib/tokens/TokenFactory.hpp" namespace ovum::compiler::lexer { diff --git a/lib/lexer/handlers/StringHandler.cpp b/lib/lexer/handlers/StringHandler.cpp index c7f2d8f..f1c31dd 100644 --- a/lib/lexer/handlers/StringHandler.cpp +++ b/lib/lexer/handlers/StringHandler.cpp @@ -2,8 +2,9 @@ #include +#include + #include "lib/lexer/LexerError.hpp" -#include "lib/tokens/TokenFactory.hpp" namespace ovum::compiler::lexer { diff --git a/lib/preprocessor/TokenProcessor.hpp b/lib/preprocessor/TokenProcessor.hpp index 99dc07f..d52a9f5 100644 --- a/lib/preprocessor/TokenProcessor.hpp +++ b/lib/preprocessor/TokenProcessor.hpp @@ -4,8 +4,9 @@ #include #include +#include + #include "PreprocessorError.hpp" -#include "lib/tokens/Token.hpp" namespace ovum::compiler::preprocessor { diff --git a/lib/preprocessor/directives_processor/TokenDirectivesProcessor.hpp b/lib/preprocessor/directives_processor/TokenDirectivesProcessor.hpp index 0317934..acdd820 100644 --- a/lib/preprocessor/directives_processor/TokenDirectivesProcessor.hpp +++ b/lib/preprocessor/directives_processor/TokenDirectivesProcessor.hpp @@ -7,10 +7,11 @@ #include #include +#include + #include "lib/preprocessor/PreprocessorError.hpp" #include "lib/preprocessor/TokenProcessor.hpp" #include "lib/preprocessor/directives_processor/handlers/DirectiveHandler.hpp" -#include "lib/tokens/Token.hpp" namespace ovum::compiler::preprocessor { diff --git a/lib/preprocessor/directives_processor/handlers/DefaultHandler.hpp b/lib/preprocessor/directives_processor/handlers/DefaultHandler.hpp index 879d15e..edf94cc 100644 --- a/lib/preprocessor/directives_processor/handlers/DefaultHandler.hpp +++ b/lib/preprocessor/directives_processor/handlers/DefaultHandler.hpp @@ -1,8 +1,9 @@ #ifndef PREPROCESSOR_DEFAULT_HANDLER_HPP_ #define PREPROCESSOR_DEFAULT_HANDLER_HPP_ +#include + #include "DirectiveHandler.hpp" -#include "lib/tokens/Token.hpp" namespace ovum::compiler::preprocessor { diff --git a/lib/preprocessor/directives_processor/handlers/DefineHandler.hpp b/lib/preprocessor/directives_processor/handlers/DefineHandler.hpp index 56c03a6..031fb36 100644 --- a/lib/preprocessor/directives_processor/handlers/DefineHandler.hpp +++ b/lib/preprocessor/directives_processor/handlers/DefineHandler.hpp @@ -1,8 +1,9 @@ #ifndef PREPROCESSOR_DEFINE_HANDLER_HPP_ #define PREPROCESSOR_DEFINE_HANDLER_HPP_ +#include + #include "DirectiveHandler.hpp" -#include "lib/tokens/Token.hpp" namespace ovum::compiler::preprocessor { diff --git a/lib/preprocessor/directives_processor/handlers/DirectiveHandler.hpp b/lib/preprocessor/directives_processor/handlers/DirectiveHandler.hpp index 6e50552..af1998c 100644 --- a/lib/preprocessor/directives_processor/handlers/DirectiveHandler.hpp +++ b/lib/preprocessor/directives_processor/handlers/DirectiveHandler.hpp @@ -7,8 +7,9 @@ #include #include +#include + #include "lib/preprocessor/PreprocessorError.hpp" -#include "lib/tokens/Token.hpp" namespace ovum::compiler::preprocessor { diff --git a/lib/preprocessor/directives_processor/handlers/ElseHandler.hpp b/lib/preprocessor/directives_processor/handlers/ElseHandler.hpp index 79cece4..597d50a 100644 --- a/lib/preprocessor/directives_processor/handlers/ElseHandler.hpp +++ b/lib/preprocessor/directives_processor/handlers/ElseHandler.hpp @@ -1,8 +1,9 @@ #ifndef PREPROCESSOR_ELSE_HANDLER_HPP_ #define PREPROCESSOR_ELSE_HANDLER_HPP_ +#include + #include "DirectiveHandler.hpp" -#include "lib/tokens/Token.hpp" namespace ovum::compiler::preprocessor { diff --git a/lib/preprocessor/directives_processor/handlers/EndifHandler.hpp b/lib/preprocessor/directives_processor/handlers/EndifHandler.hpp index bce6fae..eeb5ae4 100644 --- a/lib/preprocessor/directives_processor/handlers/EndifHandler.hpp +++ b/lib/preprocessor/directives_processor/handlers/EndifHandler.hpp @@ -1,8 +1,9 @@ #ifndef PREPROCESSOR_ENDIF_HANDLER_HPP_ #define PREPROCESSOR_ENDIF_HANDLER_HPP_ +#include + #include "DirectiveHandler.hpp" -#include "lib/tokens/Token.hpp" namespace ovum::compiler::preprocessor { diff --git a/lib/preprocessor/directives_processor/handlers/IfdefHandler.hpp b/lib/preprocessor/directives_processor/handlers/IfdefHandler.hpp index 7e9baee..86271af 100644 --- a/lib/preprocessor/directives_processor/handlers/IfdefHandler.hpp +++ b/lib/preprocessor/directives_processor/handlers/IfdefHandler.hpp @@ -1,8 +1,9 @@ #ifndef PREPROCESSOR_IFDEF_HANDLER_HPP_ #define PREPROCESSOR_IFDEF_HANDLER_HPP_ +#include + #include "DirectiveHandler.hpp" -#include "lib/tokens/Token.hpp" namespace ovum::compiler::preprocessor { diff --git a/lib/preprocessor/directives_processor/handlers/IfndefHandler.hpp b/lib/preprocessor/directives_processor/handlers/IfndefHandler.hpp index cf7a216..431be7f 100644 --- a/lib/preprocessor/directives_processor/handlers/IfndefHandler.hpp +++ b/lib/preprocessor/directives_processor/handlers/IfndefHandler.hpp @@ -1,8 +1,9 @@ #ifndef PREPROCESSOR_IFNDEF_HANDLER_HPP_ #define PREPROCESSOR_IFNDEF_HANDLER_HPP_ +#include + #include "DirectiveHandler.hpp" -#include "lib/tokens/Token.hpp" namespace ovum::compiler::preprocessor { diff --git a/lib/preprocessor/directives_processor/handlers/UndefHandler.hpp b/lib/preprocessor/directives_processor/handlers/UndefHandler.hpp index 7affa39..5b7616b 100644 --- a/lib/preprocessor/directives_processor/handlers/UndefHandler.hpp +++ b/lib/preprocessor/directives_processor/handlers/UndefHandler.hpp @@ -1,8 +1,9 @@ #ifndef PREPROCESSOR_UNDEF_HANDLER_HPP_ #define PREPROCESSOR_UNDEF_HANDLER_HPP_ +#include + #include "DirectiveHandler.hpp" -#include "lib/tokens/Token.hpp" namespace ovum::compiler::preprocessor { diff --git a/lib/preprocessor/import_processor/TokenImportProcessor.hpp b/lib/preprocessor/import_processor/TokenImportProcessor.hpp index e5fbf0a..7f6f0f7 100644 --- a/lib/preprocessor/import_processor/TokenImportProcessor.hpp +++ b/lib/preprocessor/import_processor/TokenImportProcessor.hpp @@ -9,10 +9,11 @@ #include #include +#include + #include "FileGraph.hpp" #include "lib/preprocessor/PreprocessorError.hpp" #include "lib/preprocessor/TokenProcessor.hpp" -#include "lib/tokens/Token.hpp" namespace ovum::compiler::preprocessor { diff --git a/lib/tokens/CMakeLists.txt b/lib/tokens/CMakeLists.txt deleted file mode 100644 index c56f470..0000000 --- a/lib/tokens/CMakeLists.txt +++ /dev/null @@ -1,28 +0,0 @@ -add_library(tokens STATIC - CommentToken.cpp - EofToken.cpp - IdentToken.cpp - KeywordToken.cpp - LiteralToken.cpp - NewlineToken.cpp - OperatorToken.cpp - PunctToken.cpp - TokenFactory.cpp - TokenPosition.cpp - values/BoolValue.cpp - values/CharValue.cpp - values/FloatValue.cpp - values/IntValue.cpp - values/StringValue.cpp -) - -target_include_directories(tokens - PUBLIC - ${CMAKE_SOURCE_DIR} -) - -target_include_directories(tokens - PRIVATE - ${CMAKE_CURRENT_SOURCE_DIR} -) - diff --git a/lib/tokens/CommentToken.cpp b/lib/tokens/CommentToken.cpp deleted file mode 100644 index 8f88c63..0000000 --- a/lib/tokens/CommentToken.cpp +++ /dev/null @@ -1,37 +0,0 @@ -#include "CommentToken.hpp" - -#include - -namespace ovum { - -CommentToken::CommentToken(std::string txt, const TokenPosition& position) : - text_(std::move(txt)), position_(position) { -} - -std::string CommentToken::GetStringType() const noexcept { - return "COMMENT"; -} - -std::string CommentToken::GetLexeme() const noexcept { - return text_; -} - -TokenPtr CommentToken::Clone() const { - return std::make_shared(*this); -} - -void CommentToken::Accept(TokenVisitor& visitor) const { - visitor.Visit(*this); -} - -std::string CommentToken::ToString() const { - std::ostringstream os; - os << "Token(COMMENT, '" << text_ << "', @" << position_.GetLine() << ":" << position_.GetColumn() << ")"; - return os.str(); -} - -const TokenPosition& CommentToken::GetPosition() const noexcept { - return position_; -} - -} // namespace ovum diff --git a/lib/tokens/CommentToken.hpp b/lib/tokens/CommentToken.hpp deleted file mode 100644 index c3dec6b..0000000 --- a/lib/tokens/CommentToken.hpp +++ /dev/null @@ -1,34 +0,0 @@ -#ifndef TOKENS_COMMENTTOKEN_HPP_ -#define TOKENS_COMMENTTOKEN_HPP_ - -#include - -#include "Token.hpp" -#include "TokenVisitor.hpp" - -namespace ovum { - -class CommentToken final : public Token { -public: - CommentToken(std::string txt, const TokenPosition& position); - - [[nodiscard]] std::string GetStringType() const noexcept override; - - [[nodiscard]] std::string GetLexeme() const noexcept override; - - [[nodiscard]] TokenPtr Clone() const override; - - void Accept(TokenVisitor& visitor) const override; - - [[nodiscard]] std::string ToString() const override; - - [[nodiscard]] const TokenPosition& GetPosition() const noexcept override; - -private: - std::string text_; - TokenPosition position_; -}; - -} // namespace ovum - -#endif // TOKENS_COMMENTTOKEN_HPP_ diff --git a/lib/tokens/EofToken.cpp b/lib/tokens/EofToken.cpp deleted file mode 100644 index f357e10..0000000 --- a/lib/tokens/EofToken.cpp +++ /dev/null @@ -1,36 +0,0 @@ -#include "EofToken.hpp" - -#include - -namespace ovum { - -EofToken::EofToken(const TokenPosition& position) : position_(position) { -} - -std::string EofToken::GetStringType() const noexcept { - return "EOF"; -} - -std::string EofToken::GetLexeme() const noexcept { - return ""; -} - -TokenPtr EofToken::Clone() const { - return std::make_shared(*this); -} - -void EofToken::Accept(TokenVisitor& visitor) const { - visitor.Visit(*this); -} - -std::string EofToken::ToString() const { - std::ostringstream os; - os << "Token(EOF, @" << position_.GetLine() << ":" << position_.GetColumn() << ")"; - return os.str(); -} - -const TokenPosition& EofToken::GetPosition() const noexcept { - return position_; -} - -} // namespace ovum diff --git a/lib/tokens/EofToken.hpp b/lib/tokens/EofToken.hpp deleted file mode 100644 index a36e906..0000000 --- a/lib/tokens/EofToken.hpp +++ /dev/null @@ -1,33 +0,0 @@ -#ifndef TOKENS_EOFTOKEN_HPP_ -#define TOKENS_EOFTOKEN_HPP_ - -#include - -#include "Token.hpp" -#include "TokenVisitor.hpp" - -namespace ovum { - -class EofToken final : public Token { -public: - EofToken(const TokenPosition& position); - - [[nodiscard]] std::string GetStringType() const noexcept override; - - [[nodiscard]] std::string GetLexeme() const noexcept override; - - [[nodiscard]] TokenPtr Clone() const override; - - void Accept(TokenVisitor& visitor) const override; - - [[nodiscard]] std::string ToString() const override; - - [[nodiscard]] const TokenPosition& GetPosition() const noexcept override; - -private: - TokenPosition position_; -}; - -} // namespace ovum - -#endif // TOKENS_EOFTOKEN_HPP_ diff --git a/lib/tokens/IdentToken.cpp b/lib/tokens/IdentToken.cpp deleted file mode 100644 index 98f1404..0000000 --- a/lib/tokens/IdentToken.cpp +++ /dev/null @@ -1,40 +0,0 @@ -#include "IdentToken.hpp" - -#include - -namespace ovum { - -IdentToken::IdentToken(std::string lex, const TokenPosition& position) : lexeme_(std::move(lex)), position_(position) { -} - -std::string IdentToken::GetStringType() const noexcept { - return "IDENT"; -} - -std::string IdentToken::GetLexeme() const noexcept { - return lexeme_; -} - -TokenPtr IdentToken::Clone() const { - return std::make_shared(*this); -} - -void IdentToken::Accept(TokenVisitor& visitor) const { - visitor.Visit(*this); -} - -std::string IdentToken::ToString() const { - std::ostringstream os; - os << "Token(IDENT, '" << lexeme_ << "', @" << position_.GetLine() << ":" << position_.GetColumn() << ")"; - return os.str(); -} - -const TokenPosition& IdentToken::GetPosition() const noexcept { - return position_; -} - -const std::string& IdentToken::GetName() const noexcept { - return lexeme_; -} - -} // namespace ovum diff --git a/lib/tokens/IdentToken.hpp b/lib/tokens/IdentToken.hpp deleted file mode 100644 index 0274a5c..0000000 --- a/lib/tokens/IdentToken.hpp +++ /dev/null @@ -1,36 +0,0 @@ -#ifndef TOKENS_IDENTTOKEN_HPP_ -#define TOKENS_IDENTTOKEN_HPP_ - -#include - -#include "Token.hpp" -#include "TokenVisitor.hpp" - -namespace ovum { - -class IdentToken final : public Token { -public: - IdentToken(std::string lex, const TokenPosition& position); - - [[nodiscard]] std::string GetStringType() const noexcept override; - - [[nodiscard]] std::string GetLexeme() const noexcept override; - - [[nodiscard]] TokenPtr Clone() const override; - - void Accept(TokenVisitor& visitor) const override; - - [[nodiscard]] std::string ToString() const override; - - [[nodiscard]] const std::string& GetName() const noexcept; - - [[nodiscard]] const TokenPosition& GetPosition() const noexcept override; - -private: - std::string lexeme_; - TokenPosition position_; -}; - -} // namespace ovum - -#endif // TOKENS_IDENTTOKEN_HPP_ diff --git a/lib/tokens/KeywordToken.cpp b/lib/tokens/KeywordToken.cpp deleted file mode 100644 index ca7d610..0000000 --- a/lib/tokens/KeywordToken.cpp +++ /dev/null @@ -1,37 +0,0 @@ -#include "KeywordToken.hpp" - -#include - -namespace ovum { - -KeywordToken::KeywordToken(std::string lex, const TokenPosition& position) : - lexeme_(std::move(lex)), position_(position) { -} - -std::string KeywordToken::GetStringType() const noexcept { - return "KEYWORD"; -} - -std::string KeywordToken::GetLexeme() const noexcept { - return lexeme_; -} - -TokenPtr KeywordToken::Clone() const { - return std::make_shared(*this); -} - -void KeywordToken::Accept(TokenVisitor& visitor) const { - visitor.Visit(*this); -} - -std::string KeywordToken::ToString() const { - std::ostringstream os; - os << "Token(KEYWORD, '" << lexeme_ << "', @" << position_.GetLine() << ":" << position_.GetColumn() << ")"; - return os.str(); -} - -const TokenPosition& KeywordToken::GetPosition() const noexcept { - return position_; -} - -} // namespace ovum diff --git a/lib/tokens/KeywordToken.hpp b/lib/tokens/KeywordToken.hpp deleted file mode 100644 index 05354c2..0000000 --- a/lib/tokens/KeywordToken.hpp +++ /dev/null @@ -1,34 +0,0 @@ -#ifndef TOKENS_KEYWORDTOKEN_HPP_ -#define TOKENS_KEYWORDTOKEN_HPP_ - -#include - -#include "Token.hpp" -#include "TokenVisitor.hpp" - -namespace ovum { - -class KeywordToken final : public Token { -public: - KeywordToken(std::string lex, const TokenPosition& position); - - [[nodiscard]] std::string GetStringType() const noexcept override; - - [[nodiscard]] std::string GetLexeme() const noexcept override; - - [[nodiscard]] TokenPtr Clone() const override; - - void Accept(TokenVisitor& visitor) const override; - - [[nodiscard]] std::string ToString() const override; - - [[nodiscard]] const TokenPosition& GetPosition() const noexcept override; - -private: - std::string lexeme_; - TokenPosition position_; -}; - -} // namespace ovum - -#endif // TOKENS_KEYWORDTOKEN_HPP_ diff --git a/lib/tokens/LiteralToken.cpp b/lib/tokens/LiteralToken.cpp deleted file mode 100644 index 0ed75d7..0000000 --- a/lib/tokens/LiteralToken.cpp +++ /dev/null @@ -1,53 +0,0 @@ -#include "LiteralToken.hpp" - -#include -#include - -namespace ovum { - -LiteralToken::LiteralToken(std::string rawLexeme, std::unique_ptr value, const TokenPosition& position) : - lexeme_(std::move(rawLexeme)), value_(std::move(value)), position_(position) { -} - -std::string LiteralToken::GetStringType() const noexcept { - return "LITERAL:" + value_->GetTypeName(); -} - -std::string LiteralToken::GetLexeme() const noexcept { - return lexeme_; -} - -const Value* LiteralToken::GetValue() const noexcept { - return value_.get(); -} - -Value* LiteralToken::GetValue() noexcept { - return value_.get(); -} - -TokenPtr LiteralToken::Clone() const { - std::unique_ptr vcopy = value_ ? value_->Clone() : nullptr; - return std::make_shared(lexeme_, std::move(vcopy), position_); -} - -void LiteralToken::Accept(TokenVisitor& visitor) const { - visitor.Visit(*this); -} - -std::string LiteralToken::ToString() const { - std::ostringstream os; - os << "Token(" << GetStringType() << ", '" << lexeme_ << "'"; - - if (value_) { - os << ", " << value_->ToString(); - } - - os << ", @" << position_.GetLine() << ":" << position_.GetColumn() << ")"; - return os.str(); -} - -const TokenPosition& LiteralToken::GetPosition() const noexcept { - return position_; -} - -} // namespace ovum diff --git a/lib/tokens/LiteralToken.hpp b/lib/tokens/LiteralToken.hpp deleted file mode 100644 index 9a475cc..0000000 --- a/lib/tokens/LiteralToken.hpp +++ /dev/null @@ -1,41 +0,0 @@ -#ifndef TOKENS_LITERALTOKEN_HPP_ -#define TOKENS_LITERALTOKEN_HPP_ - -#include -#include - -#include "Token.hpp" -#include "TokenVisitor.hpp" -#include "values/Value.hpp" - -namespace ovum { - -class LiteralToken final : public Token { -public: - LiteralToken(std::string rawLexeme, std::unique_ptr value, const TokenPosition& position); - - [[nodiscard]] std::string GetStringType() const noexcept override; - - [[nodiscard]] std::string GetLexeme() const noexcept override; - - [[nodiscard]] const Value* GetValue() const noexcept; - - Value* GetValue() noexcept; - - [[nodiscard]] TokenPtr Clone() const override; - - void Accept(TokenVisitor& visitor) const override; - - [[nodiscard]] std::string ToString() const override; - - [[nodiscard]] const TokenPosition& GetPosition() const noexcept override; - -private: - std::string lexeme_; - std::unique_ptr value_; - TokenPosition position_; -}; - -} // namespace ovum - -#endif // TOKENS_LITERALTOKEN_HPP_ diff --git a/lib/tokens/NewlineToken.cpp b/lib/tokens/NewlineToken.cpp deleted file mode 100644 index 99d96e3..0000000 --- a/lib/tokens/NewlineToken.cpp +++ /dev/null @@ -1,36 +0,0 @@ -#include "NewlineToken.hpp" - -#include - -namespace ovum { - -NewlineToken::NewlineToken(const TokenPosition& position) : lexeme_("\\n"), position_(position) { -} - -std::string NewlineToken::GetStringType() const noexcept { - return "NEWLINE"; -} - -std::string NewlineToken::GetLexeme() const noexcept { - return lexeme_; -} - -TokenPtr NewlineToken::Clone() const { - return std::make_shared(*this); -} - -void NewlineToken::Accept(TokenVisitor& visitor) const { - visitor.Visit(*this); -} - -std::string NewlineToken::ToString() const { - std::ostringstream os; - os << "Token(NEWLINE, '\\n', @" << position_.GetLine() << ":" << position_.GetColumn() << ")"; - return os.str(); -} - -const TokenPosition& NewlineToken::GetPosition() const noexcept { - return position_; -} - -} // namespace ovum diff --git a/lib/tokens/NewlineToken.hpp b/lib/tokens/NewlineToken.hpp deleted file mode 100644 index 72f63fa..0000000 --- a/lib/tokens/NewlineToken.hpp +++ /dev/null @@ -1,34 +0,0 @@ -#ifndef TOKENS_NEWLINETOKEN_HPP_ -#define TOKENS_NEWLINETOKEN_HPP_ - -#include - -#include "Token.hpp" -#include "TokenVisitor.hpp" - -namespace ovum { - -class NewlineToken final : public Token { -public: - NewlineToken(const TokenPosition& position); - - [[nodiscard]] std::string GetStringType() const noexcept override; - - [[nodiscard]] std::string GetLexeme() const noexcept override; - - [[nodiscard]] TokenPtr Clone() const override; - - void Accept(TokenVisitor& visitor) const override; - - [[nodiscard]] std::string ToString() const override; - - [[nodiscard]] const TokenPosition& GetPosition() const noexcept override; - -private: - std::string lexeme_; - TokenPosition position_; -}; - -} // namespace ovum - -#endif // TOKENS_NEWLINETOKEN_HPP_ diff --git a/lib/tokens/OperatorToken.cpp b/lib/tokens/OperatorToken.cpp deleted file mode 100644 index 391a527..0000000 --- a/lib/tokens/OperatorToken.cpp +++ /dev/null @@ -1,37 +0,0 @@ -#include "OperatorToken.hpp" - -#include - -namespace ovum { - -OperatorToken::OperatorToken(std::string op, const TokenPosition& position) : - lexeme_(std::move(op)), position_(position) { -} - -std::string OperatorToken::GetStringType() const noexcept { - return "OPERATOR"; -} - -std::string OperatorToken::GetLexeme() const noexcept { - return lexeme_; -} - -TokenPtr OperatorToken::Clone() const { - return std::make_shared(*this); -} - -void OperatorToken::Accept(TokenVisitor& visitor) const { - visitor.Visit(*this); -} - -std::string OperatorToken::ToString() const { - std::ostringstream os; - os << "Token(OPERATOR, '" << lexeme_ << "', @" << position_.GetLine() << ":" << position_.GetColumn() << ")"; - return os.str(); -} - -const TokenPosition& OperatorToken::GetPosition() const noexcept { - return position_; -} - -} // namespace ovum diff --git a/lib/tokens/OperatorToken.hpp b/lib/tokens/OperatorToken.hpp deleted file mode 100644 index 0e6dede..0000000 --- a/lib/tokens/OperatorToken.hpp +++ /dev/null @@ -1,35 +0,0 @@ -#ifndef TOKENS_OPERATORTOKEN_HPP_ -#define TOKENS_OPERATORTOKEN_HPP_ - -#include - -#include "Token.hpp" - -namespace ovum { - -class TokenVisitor; - -class OperatorToken final : public Token { -public: - OperatorToken(std::string op, const TokenPosition& position); - - [[nodiscard]] std::string GetStringType() const noexcept override; - - [[nodiscard]] std::string GetLexeme() const noexcept override; - - [[nodiscard]] TokenPtr Clone() const override; - - void Accept(TokenVisitor& visitor) const override; - - [[nodiscard]] std::string ToString() const override; - - [[nodiscard]] const TokenPosition& GetPosition() const noexcept override; - -private: - std::string lexeme_; - TokenPosition position_; -}; - -} // namespace ovum - -#endif // TOKENS_OPERATORTOKEN_HPP_ diff --git a/lib/tokens/PunctToken.cpp b/lib/tokens/PunctToken.cpp deleted file mode 100644 index 9815f5e..0000000 --- a/lib/tokens/PunctToken.cpp +++ /dev/null @@ -1,39 +0,0 @@ -#include -#include "PunctToken.hpp" - -namespace ovum { - -PunctToken::PunctToken(char ch, const TokenPosition& position) : lexeme_(1, ch), position_(position) { -} - -PunctToken::PunctToken(std::string punct, const TokenPosition& position) : - lexeme_(std::move(punct)), position_(position) { -} - -std::string PunctToken::GetStringType() const noexcept { - return "PUNCT"; -} - -std::string PunctToken::GetLexeme() const noexcept { - return lexeme_; -} - -TokenPtr PunctToken::Clone() const { - return std::make_shared(*this); -} - -void PunctToken::Accept(TokenVisitor& visitor) const { - visitor.Visit(*this); -} - -std::string PunctToken::ToString() const { - std::ostringstream os; - os << "Token(PUNCT, '" << lexeme_ << "', @" << position_.GetLine() << ":" << position_.GetColumn() << ")"; - return os.str(); -} - -const TokenPosition& PunctToken::GetPosition() const noexcept { - return position_; -} - -} // namespace ovum diff --git a/lib/tokens/PunctToken.hpp b/lib/tokens/PunctToken.hpp deleted file mode 100644 index 89de415..0000000 --- a/lib/tokens/PunctToken.hpp +++ /dev/null @@ -1,36 +0,0 @@ -#ifndef TOKENS_PUNCTTOKEN_HPP_ -#define TOKENS_PUNCTTOKEN_HPP_ - -#include - -#include "Token.hpp" -#include "TokenVisitor.hpp" - -namespace ovum { - -class PunctToken final : public Token { -public: - PunctToken(char ch, const TokenPosition& position); - - PunctToken(std::string punct, const TokenPosition& position); - - [[nodiscard]] std::string GetStringType() const noexcept override; - - [[nodiscard]] std::string GetLexeme() const noexcept override; - - [[nodiscard]] TokenPtr Clone() const override; - - void Accept(TokenVisitor& visitor) const override; - - [[nodiscard]] std::string ToString() const override; - - [[nodiscard]] const TokenPosition& GetPosition() const noexcept override; - -private: - std::string lexeme_; - TokenPosition position_; -}; - -} // namespace ovum - -#endif // TOKENS_PUNCTTOKEN_HPP_ diff --git a/lib/tokens/Token.hpp b/lib/tokens/Token.hpp deleted file mode 100644 index a1a85bf..0000000 --- a/lib/tokens/Token.hpp +++ /dev/null @@ -1,33 +0,0 @@ -#ifndef TOKENS_TOKEN_HPP_ -#define TOKENS_TOKEN_HPP_ - -#include -#include - -#include "TokenPosition.hpp" -#include "TokenVisitor.hpp" - -namespace ovum { - -class Token { // NOLINT(cppcoreguidelines-special-member-functions) -public: - virtual ~Token() = default; - - [[nodiscard]] virtual std::string GetStringType() const = 0; - - [[nodiscard]] virtual std::string GetLexeme() const = 0; - - [[nodiscard]] virtual std::shared_ptr Clone() const = 0; - - virtual void Accept(TokenVisitor& v) const = 0; - - [[nodiscard]] virtual std::string ToString() const = 0; - - [[nodiscard]] virtual const TokenPosition& GetPosition() const noexcept = 0; -}; - -using TokenPtr = std::shared_ptr; - -} // namespace ovum - -#endif // TOKENS_TOKEN_HPP_ diff --git a/lib/tokens/TokenFactory.cpp b/lib/tokens/TokenFactory.cpp deleted file mode 100644 index 114b77d..0000000 --- a/lib/tokens/TokenFactory.cpp +++ /dev/null @@ -1,76 +0,0 @@ -#include "TokenFactory.hpp" - -#include "CommentToken.hpp" -#include "EofToken.hpp" -#include "IdentToken.hpp" -#include "KeywordToken.hpp" -#include "LiteralToken.hpp" -#include "NewlineToken.hpp" -#include "OperatorToken.hpp" -#include "PunctToken.hpp" -#include "values/BoolValue.hpp" -#include "values/CharValue.hpp" -#include "values/FloatValue.hpp" -#include "values/IntValue.hpp" -#include "values/StringValue.hpp" - -namespace ovum { - -namespace TokenFactory { - -TokenPtr MakeIdent(std::string lex, int32_t line, int32_t col) { - return std::make_shared(std::move(lex), TokenPosition(line, col)); -} - -TokenPtr MakeKeyword(std::string lex, int32_t line, int32_t col) { - return std::make_shared(std::move(lex), TokenPosition(line, col)); -} - -TokenPtr MakeOperator(std::string lex, int32_t line, int32_t col) { - return std::make_shared(std::move(lex), TokenPosition(line, col)); -} - -TokenPtr MakePunct(char ch, int32_t line, int32_t col) { - return std::make_shared(ch, TokenPosition(line, col)); -} - -TokenPtr MakePunct(std::string lex, int32_t line, int32_t col) { - return std::make_shared(std::move(lex), TokenPosition(line, col)); -} - -TokenPtr MakeNewline(int32_t line, int32_t col) { - return std::make_shared(TokenPosition(line, col)); -} - -TokenPtr MakeComment(std::string text, int32_t line, int32_t col) { - return std::make_shared(std::move(text), TokenPosition(line, col)); -} - -TokenPtr MakeEof(int32_t line, int32_t col) { - return std::make_shared(TokenPosition(line, col)); -} - -TokenPtr MakeIntLiteral(std::string raw, int64_t v, int32_t line, int32_t col) { - return std::make_shared(std::move(raw), std::make_unique(v), TokenPosition(line, col)); -} - -TokenPtr MakeFloatLiteral(std::string raw, double v, int32_t line, int32_t col) { - return std::make_shared(std::move(raw), std::make_unique(v), TokenPosition(line, col)); -} - -TokenPtr MakeStringLiteral(std::string raw, std::string s, int32_t line, int32_t col) { - return std::make_shared( - std::move(raw), std::make_unique(std::move(s)), TokenPosition(line, col)); -} - -TokenPtr MakeCharLiteral(std::string raw, char c, int32_t line, int32_t col) { - return std::make_shared(std::move(raw), std::make_unique(c), TokenPosition(line, col)); -} - -TokenPtr MakeBoolLiteral(std::string raw, bool b, int32_t line, int32_t col) { - return std::make_shared(std::move(raw), std::make_unique(b), TokenPosition(line, col)); -} - -} // namespace TokenFactory - -} // namespace ovum diff --git a/lib/tokens/TokenFactory.hpp b/lib/tokens/TokenFactory.hpp deleted file mode 100644 index e2501ae..0000000 --- a/lib/tokens/TokenFactory.hpp +++ /dev/null @@ -1,38 +0,0 @@ -#ifndef TOKENS_TOKENFACTORY_HPP_ -#define TOKENS_TOKENFACTORY_HPP_ - -#include - -#include "Token.hpp" - -namespace ovum::TokenFactory { // NOLINT(readability-identifier-naming) - -TokenPtr MakeIdent(std::string lex, int32_t line, int32_t col); - -TokenPtr MakeKeyword(std::string lex, int32_t line, int32_t col); - -TokenPtr MakeOperator(std::string lex, int32_t line, int32_t col); - -TokenPtr MakePunct(char ch, int32_t line, int32_t col); - -TokenPtr MakePunct(std::string lex, int32_t line, int32_t col); - -TokenPtr MakeNewline(int32_t line, int32_t col); - -TokenPtr MakeComment(std::string text, int32_t line, int32_t col); - -TokenPtr MakeEof(int32_t line, int32_t col); - -TokenPtr MakeIntLiteral(std::string raw, int64_t v, int32_t line, int32_t col); - -TokenPtr MakeFloatLiteral(std::string raw, double v, int32_t line, int32_t col); - -TokenPtr MakeStringLiteral(std::string raw, std::string s, int32_t line, int32_t col); - -TokenPtr MakeCharLiteral(std::string raw, char c, int32_t line, int32_t col); - -TokenPtr MakeBoolLiteral(std::string raw, bool b, int32_t line, int32_t col); - -} // namespace ovum::TokenFactory - -#endif // TOKENS_TOKENFACTORY_HPP_ diff --git a/lib/tokens/TokenPosition.cpp b/lib/tokens/TokenPosition.cpp deleted file mode 100644 index 4d8da5c..0000000 --- a/lib/tokens/TokenPosition.cpp +++ /dev/null @@ -1,16 +0,0 @@ -#include "TokenPosition.hpp" - -namespace ovum { - -TokenPosition::TokenPosition(int32_t line, int32_t column) : line_(line), column_(column) { -} - -int32_t TokenPosition::GetLine() const noexcept { - return line_; -} - -int32_t TokenPosition::GetColumn() const noexcept { - return column_; -} - -} // namespace ovum diff --git a/lib/tokens/TokenPosition.hpp b/lib/tokens/TokenPosition.hpp deleted file mode 100644 index 9a27154..0000000 --- a/lib/tokens/TokenPosition.hpp +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef TOKENS_TOKENPOSITION_HPP_ -#define TOKENS_TOKENPOSITION_HPP_ - -#include - -namespace ovum { - -class TokenPosition { -public: - TokenPosition(int32_t line, int32_t column); - - [[nodiscard]] int32_t GetLine() const noexcept; - - [[nodiscard]] int32_t GetColumn() const noexcept; - -private: - int32_t line_; - int32_t column_; -}; - -} // namespace ovum - -#endif // TOKENS_TOKENPOSITION_HPP_ diff --git a/lib/tokens/TokenVisitor.hpp b/lib/tokens/TokenVisitor.hpp deleted file mode 100644 index c721c22..0000000 --- a/lib/tokens/TokenVisitor.hpp +++ /dev/null @@ -1,38 +0,0 @@ -#ifndef TOKENS_TOKENVISITOR_HPP_ -#define TOKENS_TOKENVISITOR_HPP_ - -namespace ovum { - -class EofToken; -class CommentToken; -class NewlineToken; -class PunctToken; -class OperatorToken; -class LiteralToken; -class KeywordToken; -class IdentToken; - -class TokenVisitor { // NOLINT(cppcoreguidelines-special-member-functions) -public: - virtual ~TokenVisitor() = default; - - virtual void Visit(const IdentToken& t) = 0; - - virtual void Visit(const KeywordToken& t) = 0; - - virtual void Visit(const LiteralToken& t) = 0; - - virtual void Visit(const OperatorToken& t) = 0; - - virtual void Visit(const PunctToken& t) = 0; - - virtual void Visit(const NewlineToken& t) = 0; - - virtual void Visit(const CommentToken& t) = 0; - - virtual void Visit(const EofToken& t) = 0; -}; - -} // namespace ovum - -#endif // TOKENS_TOKENVISITOR_HPP_ diff --git a/lib/tokens/values/BoolValue.cpp b/lib/tokens/values/BoolValue.cpp deleted file mode 100644 index d60f1a0..0000000 --- a/lib/tokens/values/BoolValue.cpp +++ /dev/null @@ -1,20 +0,0 @@ -#include "BoolValue.hpp" - -namespace ovum { - -BoolValue::BoolValue(bool b) : v(b) { -} - -std::unique_ptr BoolValue::Clone() const { - return std::make_unique(v); -} - -std::string BoolValue::ToString() const { - return v ? "true" : "false"; -} - -std::string BoolValue::GetTypeName() const { - return "Bool"; -} - -} // namespace ovum diff --git a/lib/tokens/values/BoolValue.hpp b/lib/tokens/values/BoolValue.hpp deleted file mode 100644 index fca95f9..0000000 --- a/lib/tokens/values/BoolValue.hpp +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef TOKENS_BOOLVALUE_HPP_ -#define TOKENS_BOOLVALUE_HPP_ - -#include -#include - -#include "Value.hpp" - -namespace ovum { - -class BoolValue : public Value { -public: - explicit BoolValue(bool b); - - [[nodiscard]] std::unique_ptr Clone() const override; - - [[nodiscard]] std::string ToString() const override; - - [[nodiscard]] std::string GetTypeName() const override; - -private: - bool v; -}; - -} // namespace ovum - -#endif // TOKENS_BOOLVALUE_HPP_ diff --git a/lib/tokens/values/CharValue.cpp b/lib/tokens/values/CharValue.cpp deleted file mode 100644 index 78146ef..0000000 --- a/lib/tokens/values/CharValue.cpp +++ /dev/null @@ -1,20 +0,0 @@ -#include "CharValue.hpp" - -namespace ovum { - -CharValue::CharValue(char c) : v(c) { -} - -std::unique_ptr CharValue::Clone() const { - return std::make_unique(v); -} - -std::string CharValue::ToString() const { - return "'" + std::string(1, v) + "'"; -} - -std::string CharValue::GetTypeName() const { - return "Char"; -} - -} // namespace ovum diff --git a/lib/tokens/values/CharValue.hpp b/lib/tokens/values/CharValue.hpp deleted file mode 100644 index 263fca8..0000000 --- a/lib/tokens/values/CharValue.hpp +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef TOKENS_CHARVALUE_HPP_ -#define TOKENS_CHARVALUE_HPP_ - -#include -#include - -#include "Value.hpp" - -namespace ovum { - -class CharValue : public Value { -public: - explicit CharValue(char c); - - [[nodiscard]] std::unique_ptr Clone() const override; - - [[nodiscard]] std::string ToString() const override; - - [[nodiscard]] std::string GetTypeName() const override; - -private: - char v; -}; - -} // namespace ovum - -#endif // TOKENS_CHARVALUE_HPP_ diff --git a/lib/tokens/values/FloatValue.cpp b/lib/tokens/values/FloatValue.cpp deleted file mode 100644 index 686ca23..0000000 --- a/lib/tokens/values/FloatValue.cpp +++ /dev/null @@ -1,24 +0,0 @@ -#include "FloatValue.hpp" - -#include - -namespace ovum { - -FloatValue::FloatValue(double x) : v(x) { -} - -std::unique_ptr FloatValue::Clone() const { - return std::make_unique(v); -} - -std::string FloatValue::ToString() const { - std::ostringstream os; - os << v; - return os.str(); -} - -std::string FloatValue::GetTypeName() const { - return "Float"; -} - -} // namespace ovum diff --git a/lib/tokens/values/FloatValue.hpp b/lib/tokens/values/FloatValue.hpp deleted file mode 100644 index 92b860f..0000000 --- a/lib/tokens/values/FloatValue.hpp +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef TOKENS_FLOATVALUE_HPP_ -#define TOKENS_FLOATVALUE_HPP_ - -#include -#include - -#include "Value.hpp" - -namespace ovum { - -class FloatValue : public Value { -public: - explicit FloatValue(double x); - - [[nodiscard]] std::unique_ptr Clone() const override; - - [[nodiscard]] std::string ToString() const override; - - [[nodiscard]] std::string GetTypeName() const override; - -private: - double v; -}; - -} // namespace ovum - -#endif // TOKENS_FLOATVALUE_HPP_ diff --git a/lib/tokens/values/IntValue.cpp b/lib/tokens/values/IntValue.cpp deleted file mode 100644 index 3cabf55..0000000 --- a/lib/tokens/values/IntValue.cpp +++ /dev/null @@ -1,20 +0,0 @@ -#include "IntValue.hpp" - -namespace ovum { - -IntValue::IntValue(int64_t x) : v(x) { -} - -std::unique_ptr IntValue::Clone() const { - return std::make_unique(v); -} - -std::string IntValue::ToString() const { - return std::to_string(v); -} - -std::string IntValue::GetTypeName() const { - return "Int"; -} - -} // namespace ovum diff --git a/lib/tokens/values/IntValue.hpp b/lib/tokens/values/IntValue.hpp deleted file mode 100644 index 3df8612..0000000 --- a/lib/tokens/values/IntValue.hpp +++ /dev/null @@ -1,28 +0,0 @@ -#ifndef TOKENS_INTVALUE_HPP_ -#define TOKENS_INTVALUE_HPP_ - -#include -#include -#include - -#include "Value.hpp" - -namespace ovum { - -class IntValue : public Value { -public: - explicit IntValue(int64_t x); - - [[nodiscard]] std::unique_ptr Clone() const override; - - [[nodiscard]] std::string ToString() const override; - - [[nodiscard]] std::string GetTypeName() const override; - -private: - int64_t v; -}; - -} // namespace ovum - -#endif // TOKENS_INTVALUE_HPP_ diff --git a/lib/tokens/values/StringValue.cpp b/lib/tokens/values/StringValue.cpp deleted file mode 100644 index 75f84f0..0000000 --- a/lib/tokens/values/StringValue.cpp +++ /dev/null @@ -1,22 +0,0 @@ -#include "StringValue.hpp" - -#include - -namespace ovum { - -StringValue::StringValue(std::string s) : v(std::move(s)) { -} - -std::unique_ptr StringValue::Clone() const { - return std::make_unique(v); -} - -std::string StringValue::ToString() const { - return "\"" + v + "\""; -} - -std::string StringValue::GetTypeName() const { - return "String"; -} - -} // namespace ovum diff --git a/lib/tokens/values/StringValue.hpp b/lib/tokens/values/StringValue.hpp deleted file mode 100644 index 8649b41..0000000 --- a/lib/tokens/values/StringValue.hpp +++ /dev/null @@ -1,27 +0,0 @@ -#ifndef TOKENS_STRINGVALUE_HPP_ -#define TOKENS_STRINGVALUE_HPP_ - -#include -#include - -#include "Value.hpp" - -namespace ovum { - -class StringValue : public Value { -public: - explicit StringValue(std::string s); - - [[nodiscard]] std::unique_ptr Clone() const override; - - [[nodiscard]] std::string ToString() const override; - - [[nodiscard]] std::string GetTypeName() const override; - -private: - std::string v; -}; - -} // namespace ovum - -#endif // TOKENS_STRINGVALUE_HPP_ diff --git a/lib/tokens/values/Value.hpp b/lib/tokens/values/Value.hpp deleted file mode 100644 index d4c572c..0000000 --- a/lib/tokens/values/Value.hpp +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef TOKENS_VALUE_HPP_ -#define TOKENS_VALUE_HPP_ - -#include -#include - -namespace ovum { - -class Value { // NOLINT(cppcoreguidelines-special-member-functions) -public: - virtual ~Value() = default; - - [[nodiscard]] virtual std::unique_ptr Clone() const = 0; - - [[nodiscard]] virtual std::string ToString() const = 0; - - [[nodiscard]] virtual std::string GetTypeName() const = 0; -}; - -} // namespace ovum - -#endif // TOKENS_VALUE_HPP_ diff --git a/tests/test_suites/LexerUnitTestSuite.hpp b/tests/test_suites/LexerUnitTestSuite.hpp index 4c0c3fc..217f296 100644 --- a/tests/test_suites/LexerUnitTestSuite.hpp +++ b/tests/test_suites/LexerUnitTestSuite.hpp @@ -2,7 +2,8 @@ #define OVUMC_LEXERUNITTESTSUITE_HPP #include -#include "lib/tokens/Token.hpp" + +#include struct LexerUnitTestSuite : public testing::Test { public: diff --git a/tests/test_suites/PreprocessorUnitTestSuite.hpp b/tests/test_suites/PreprocessorUnitTestSuite.hpp index c56556e..0c1d7a6 100644 --- a/tests/test_suites/PreprocessorUnitTestSuite.hpp +++ b/tests/test_suites/PreprocessorUnitTestSuite.hpp @@ -6,8 +6,9 @@ #include #include +#include + #include "lib/preprocessor/PreprocessorError.hpp" -#include "lib/tokens/Token.hpp" namespace ovum::compiler::preprocessor {