diff --git a/cmake/IncludeExternalLibraries.cmake b/cmake/IncludeExternalLibraries.cmake index d002ab7..95e121b 100644 --- a/cmake/IncludeExternalLibraries.cmake +++ b/cmake/IncludeExternalLibraries.cmake @@ -11,3 +11,12 @@ FetchContent_Declare( ) FetchContent_MakeAvailable(googletest) + +# Ovum Common +FetchContent_Declare( + ovumcommon + GIT_REPOSITORY https://github.com/Ovum-Programming-Language/OvumCommon.git + GIT_TAG main +) + +FetchContent_MakeAvailable(ovumcommon) diff --git a/lib/lexer/CMakeLists.txt b/lib/lexer/CMakeLists.txt index 4a04cbe..648365d 100644 --- a/lib/lexer/CMakeLists.txt +++ b/lib/lexer/CMakeLists.txt @@ -13,21 +13,6 @@ add_library(lexer STATIC handlers/SlashHandler.cpp handlers/StringHandler.cpp handlers/WhitespaceHandler.cpp - tokens/CommentToken.cpp - tokens/EofToken.cpp - tokens/IdentToken.cpp - tokens/KeywordToken.cpp - tokens/LiteralToken.cpp - tokens/NewlineToken.cpp - tokens/OperatorToken.cpp - tokens/PunctToken.cpp - values/BoolValue.cpp - values/CharValue.cpp - values/FloatValue.cpp - values/IntValue.cpp - values/StringValue.cpp - tokens/TokenFactory.cpp - tokens/TokenPosition.cpp ) target_include_directories(lexer @@ -39,3 +24,7 @@ target_include_directories(lexer PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ) + +target_link_libraries(lexer PUBLIC + tokens +) diff --git a/lib/lexer/Lexer.cpp b/lib/lexer/Lexer.cpp index 46a84c7..da817ac 100644 --- a/lib/lexer/Lexer.cpp +++ b/lib/lexer/Lexer.cpp @@ -3,6 +3,8 @@ #include #include +#include + #include "handlers/CharHandler.hpp" #include "handlers/ColonHandler.hpp" #include "handlers/DefaultHandler.hpp" @@ -16,7 +18,7 @@ #include "handlers/StringHandler.hpp" #include "handlers/WhitespaceHandler.hpp" -#include "tokens/TokenFactory.hpp" +namespace ovum::compiler::lexer { namespace { constexpr const char* kOperatorChars = "+-*%<>=!&|^~?"; @@ -27,7 +29,7 @@ Lexer::Lexer(std::string_view src, bool keep_comments) : wrapper_(src, keep_comments), handlers_(MakeDefaultHandlers()), default_handler_(MakeDefaultHandler()) { } -std::vector Lexer::Tokenize() { +std::expected, LexerError> Lexer::Tokenize() { std::vector tokens; tokens.reserve(kDefaultTokenReserve); @@ -41,7 +43,13 @@ std::vector Lexer::Tokenize() { current_handler = default_handler_.get(); } - OptToken maybe_token = current_handler->Scan(wrapper_); + auto maybe_token_result = current_handler->Scan(wrapper_); + + if (!maybe_token_result) { + return std::unexpected(maybe_token_result.error()); + } + + OptToken maybe_token = maybe_token_result.value(); if (maybe_token && *maybe_token) { tokens.push_back(std::move(*maybe_token)); @@ -105,3 +113,5 @@ std::array, kDefaultTokenReserve> Lexer::MakeDefaultHan std::unique_ptr Lexer::MakeDefaultHandler() { return std::make_unique(); } + +} // namespace ovum::compiler::lexer diff --git a/lib/lexer/Lexer.hpp b/lib/lexer/Lexer.hpp index 39bfdd0..9eaced0 100644 --- a/lib/lexer/Lexer.hpp +++ b/lib/lexer/Lexer.hpp @@ -1,22 +1,30 @@ -#ifndef LEXER_HPP_ -#define LEXER_HPP_ +#ifndef LEXER_LEXER_HPP_ +#define LEXER_LEXER_HPP_ #include #include +#include #include #include #include +#include + +#include "LexerError.hpp" #include "SourceCodeWrapper.hpp" #include "handlers/Handler.hpp" +namespace ovum::compiler::lexer { + +using TokenPtr = ovum::TokenPtr; + constexpr std::size_t kDefaultTokenReserve = 256; class Lexer { public: explicit Lexer(std::string_view src, bool keep_comments = false); - std::vector Tokenize(); + std::expected, LexerError> Tokenize(); void SetHandler(unsigned char ch, std::unique_ptr handler); @@ -32,4 +40,6 @@ class Lexer { std::unique_ptr default_handler_; }; -#endif // LEXER_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_LEXER_HPP_ diff --git a/lib/lexer/LexerError.hpp b/lib/lexer/LexerError.hpp index 0c55e8a..2c859cf 100644 --- a/lib/lexer/LexerError.hpp +++ b/lib/lexer/LexerError.hpp @@ -1,11 +1,15 @@ -#ifndef LEXERERROR_HPP_ -#define LEXERERROR_HPP_ +#ifndef LEXER_LEXERERROR_HPP_ +#define LEXER_LEXERERROR_HPP_ #include +namespace ovum::compiler::lexer { + class LexerError : public std::runtime_error { public: using std::runtime_error::runtime_error; }; -#endif // LEXERERROR_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_LEXERERROR_HPP_ diff --git a/lib/lexer/SourceCodeWrapper.cpp b/lib/lexer/SourceCodeWrapper.cpp index a5033bc..ec76a9d 100644 --- a/lib/lexer/SourceCodeWrapper.cpp +++ b/lib/lexer/SourceCodeWrapper.cpp @@ -2,6 +2,8 @@ #include +namespace ovum::compiler::lexer { + const std::unordered_set SourceCodeWrapper::kKeywords = { "fun", "class", "interface", "var", "override", "pure", "if", "else", "continue", "break", "for", "while", "return", "unsafe", "val", "static", "public", "private", @@ -121,3 +123,5 @@ bool SourceCodeWrapper::IsKeyword(std::string_view s) { bool SourceCodeWrapper::IsMultiOp(std::string_view s) { return kMultiOps.contains(s); } + +} // namespace ovum::compiler::lexer diff --git a/lib/lexer/SourceCodeWrapper.hpp b/lib/lexer/SourceCodeWrapper.hpp index c6094c0..207c60d 100644 --- a/lib/lexer/SourceCodeWrapper.hpp +++ b/lib/lexer/SourceCodeWrapper.hpp @@ -1,5 +1,5 @@ -#ifndef SOURCECODEWRAPPER_HPP_ -#define SOURCECODEWRAPPER_HPP_ +#ifndef LEXER_SOURCECODEWRAPPER_HPP_ +#define LEXER_SOURCECODEWRAPPER_HPP_ #include #include @@ -7,6 +7,8 @@ #include #include +namespace ovum::compiler::lexer { + class SourceCodeWrapper { public: SourceCodeWrapper(std::string_view src, bool keep_comments = false); @@ -53,4 +55,6 @@ class SourceCodeWrapper { static const std::unordered_set kMultiOps; }; -#endif // SOURCECODEWRAPPER_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_SOURCECODEWRAPPER_HPP_ diff --git a/lib/lexer/handlers/CharHandler.cpp b/lib/lexer/handlers/CharHandler.cpp index 7f7a54f..3e8b2fe 100644 --- a/lib/lexer/handlers/CharHandler.cpp +++ b/lib/lexer/handlers/CharHandler.cpp @@ -1,9 +1,12 @@ #include "CharHandler.hpp" +#include + #include "lib/lexer/LexerError.hpp" -#include "tokens/TokenFactory.hpp" -OptToken CharHandler::Scan(SourceCodeWrapper& wrapper) { +namespace ovum::compiler::lexer { + +std::expected CharHandler::Scan(SourceCodeWrapper& wrapper) { std::string raw; raw.push_back('\''); char val = '\0'; @@ -30,7 +33,7 @@ OptToken CharHandler::Scan(SourceCodeWrapper& wrapper) { val = '\0'; break; default: - throw LexerError(std::string("Unknown escape in char literal: \\") + e); + return std::unexpected(LexerError(std::string("Unknown escape in char literal: \\") + e)); } } else { char c = wrapper.Advance(); @@ -39,20 +42,22 @@ OptToken CharHandler::Scan(SourceCodeWrapper& wrapper) { } if (wrapper.IsAtEnd()) { - throw LexerError("Unterminated char literal"); + return std::unexpected(LexerError("Unterminated char literal")); } if (wrapper.Peek() == '\n') { - throw LexerError("Newline in char literal"); + return std::unexpected(LexerError("Newline in char literal")); } if (wrapper.Peek() == '\'') { wrapper.Advance(); raw.push_back('\''); } else { - throw LexerError("Too many characters in char literal"); + return std::unexpected(LexerError("Too many characters in char literal")); } return std::make_optional( TokenFactory::MakeCharLiteral(std::move(raw), val, wrapper.GetLine(), wrapper.GetTokenCol())); } + +} // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/CharHandler.hpp b/lib/lexer/handlers/CharHandler.hpp index 6d07ebf..de6981d 100644 --- a/lib/lexer/handlers/CharHandler.hpp +++ b/lib/lexer/handlers/CharHandler.hpp @@ -1,11 +1,15 @@ -#ifndef CHARHANDLER_HPP_ -#define CHARHANDLER_HPP_ +#ifndef LEXER_CHARHANDLER_HPP_ +#define LEXER_CHARHANDLER_HPP_ #include "Handler.hpp" +namespace ovum::compiler::lexer { + class CharHandler : public Handler { public: - OptToken Scan(SourceCodeWrapper& wrapper) override; + std::expected Scan(SourceCodeWrapper& wrapper) override; }; -#endif // CHARHANDLER_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_CHARHANDLER_HPP_ diff --git a/lib/lexer/handlers/ColonHandler.cpp b/lib/lexer/handlers/ColonHandler.cpp index 6807b91..fec0f3c 100644 --- a/lib/lexer/handlers/ColonHandler.cpp +++ b/lib/lexer/handlers/ColonHandler.cpp @@ -1,7 +1,10 @@ #include "ColonHandler.hpp" -#include "lib/lexer/tokens/TokenFactory.hpp" -OptToken ColonHandler::Scan(SourceCodeWrapper& wrapper) { +#include + +namespace ovum::compiler::lexer { + +std::expected ColonHandler::Scan(SourceCodeWrapper& wrapper) { std::string op; op.push_back(wrapper.CurrentChar()); @@ -18,3 +21,5 @@ OptToken ColonHandler::Scan(SourceCodeWrapper& wrapper) { return std::make_optional(TokenFactory::MakePunct(':', wrapper.GetLine(), wrapper.GetTokenCol())); } + +} // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/ColonHandler.hpp b/lib/lexer/handlers/ColonHandler.hpp index 9184ff5..602c473 100644 --- a/lib/lexer/handlers/ColonHandler.hpp +++ b/lib/lexer/handlers/ColonHandler.hpp @@ -1,10 +1,14 @@ -#ifndef COLONHANDLER_HPP_ -#define COLONHANDLER_HPP_ +#ifndef LEXER_COLONHANDLER_HPP_ +#define LEXER_COLONHANDLER_HPP_ #include "Handler.hpp" +namespace ovum::compiler::lexer { + class ColonHandler : public Handler { public: - OptToken Scan(SourceCodeWrapper& wrapper) override; + std::expected Scan(SourceCodeWrapper& wrapper) override; }; -#endif // COLONHANDLER_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_COLONHANDLER_HPP_ diff --git a/lib/lexer/handlers/DefaultHandler.cpp b/lib/lexer/handlers/DefaultHandler.cpp index e87bc8c..8c9c708 100644 --- a/lib/lexer/handlers/DefaultHandler.cpp +++ b/lib/lexer/handlers/DefaultHandler.cpp @@ -2,7 +2,11 @@ #include "lib/lexer/LexerError.hpp" -OptToken DefaultHandler::Scan(SourceCodeWrapper& wrapper) { +namespace ovum::compiler::lexer { + +std::expected DefaultHandler::Scan(SourceCodeWrapper& wrapper) { char c = wrapper.CurrentChar(); - throw LexerError(std::string("Unexpected character: ") + c); + return std::unexpected(LexerError(std::string("Unexpected character: ") + c)); } + +} // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/DefaultHandler.hpp b/lib/lexer/handlers/DefaultHandler.hpp index 9374a84..860b1a7 100644 --- a/lib/lexer/handlers/DefaultHandler.hpp +++ b/lib/lexer/handlers/DefaultHandler.hpp @@ -1,11 +1,15 @@ -#ifndef DEFAULTHANDLER_HPP_ -#define DEFAULTHANDLER_HPP_ +#ifndef LEXER_DEFAULTHANDLER_HPP_ +#define LEXER_DEFAULTHANDLER_HPP_ #include "Handler.hpp" +namespace ovum::compiler::lexer { + class DefaultHandler : public Handler { public: - OptToken Scan(SourceCodeWrapper& wrapper) override; + std::expected Scan(SourceCodeWrapper& wrapper) override; }; -#endif // DEFAULTHANDLER_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_DEFAULTHANDLER_HPP_ diff --git a/lib/lexer/handlers/DotCompositeHandler.cpp b/lib/lexer/handlers/DotCompositeHandler.cpp index df0f202..dd7979a 100644 --- a/lib/lexer/handlers/DotCompositeHandler.cpp +++ b/lib/lexer/handlers/DotCompositeHandler.cpp @@ -1,22 +1,26 @@ #include "DotCompositeHandler.hpp" +namespace ovum::compiler::lexer { + DotCompositeHandler::DotCompositeHandler() : num_(std::make_unique()), op_(std::make_unique()), punct_(std::make_unique()) { } -OptToken DotCompositeHandler::Scan(SourceCodeWrapper& w) { +std::expected DotCompositeHandler::Scan(SourceCodeWrapper& w) { if (std::isdigit(static_cast(w.Peek()))) { return num_->Scan(w); } - if (auto t = op_->Scan(w)) { - return t; + auto op_result = op_->Scan(w); + if (!op_result) { + return op_result; } - - if (punct_) { - return punct_->Scan(w); + if (op_result.value()) { + return op_result; } - return std::nullopt; + return punct_->Scan(w); } + +} // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/DotCompositeHandler.hpp b/lib/lexer/handlers/DotCompositeHandler.hpp index 4eb8a1c..301fffc 100644 --- a/lib/lexer/handlers/DotCompositeHandler.hpp +++ b/lib/lexer/handlers/DotCompositeHandler.hpp @@ -1,16 +1,18 @@ -#ifndef DOTCOMPOSITEHANDLER_HPP_ -#define DOTCOMPOSITEHANDLER_HPP_ +#ifndef LEXER_DOTCOMPOSITEHANDLER_HPP_ +#define LEXER_DOTCOMPOSITEHANDLER_HPP_ #include "Handler.hpp" #include "NumberHandler.hpp" #include "OperatorHandler.hpp" #include "PunctHandler.hpp" +namespace ovum::compiler::lexer { + class DotCompositeHandler : public Handler { public: DotCompositeHandler(); - OptToken Scan(SourceCodeWrapper& wrapper) override; + std::expected Scan(SourceCodeWrapper& wrapper) override; private: std::unique_ptr num_; @@ -18,4 +20,6 @@ class DotCompositeHandler : public Handler { std::unique_ptr punct_; }; -#endif // DOTCOMPOSITEHANDLER_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_DOTCOMPOSITEHANDLER_HPP_ diff --git a/lib/lexer/handlers/Handler.hpp b/lib/lexer/handlers/Handler.hpp index bd960c1..6638e64 100644 --- a/lib/lexer/handlers/Handler.hpp +++ b/lib/lexer/handlers/Handler.hpp @@ -1,18 +1,26 @@ -#ifndef HANDLER_HPP_ -#define HANDLER_HPP_ +#ifndef LEXER_HANDLER_HPP_ +#define LEXER_HANDLER_HPP_ +#include #include +#include + +#include "lib/lexer/LexerError.hpp" #include "lib/lexer/SourceCodeWrapper.hpp" -#include "lib/lexer/tokens/Token.hpp" +namespace ovum::compiler::lexer { + +using TokenPtr = ovum::TokenPtr; using OptToken = std::optional; class Handler { // NOLINT(cppcoreguidelines-special-member-functions) public: virtual ~Handler() = default; - virtual OptToken Scan(SourceCodeWrapper& wrapper) = 0; + virtual std::expected Scan(SourceCodeWrapper& wrapper) = 0; }; -#endif // HANDLER_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_HANDLER_HPP_ diff --git a/lib/lexer/handlers/IdentifierHandler.cpp b/lib/lexer/handlers/IdentifierHandler.cpp index bebe045..23e4afe 100644 --- a/lib/lexer/handlers/IdentifierHandler.cpp +++ b/lib/lexer/handlers/IdentifierHandler.cpp @@ -3,10 +3,13 @@ #include #include +#include + #include "LexerError.hpp" -#include "lib/lexer/tokens/TokenFactory.hpp" -OptToken IdentifierHandler::Scan(SourceCodeWrapper& wrapper) { +namespace ovum::compiler::lexer { + +std::expected IdentifierHandler::Scan(SourceCodeWrapper& wrapper) { std::string s; s.push_back(wrapper.CurrentChar()); wrapper.ConsumeWhile(s, [](char ch) { return std::isalnum(static_cast(ch)) || ch == '_'; }); @@ -31,8 +34,10 @@ OptToken IdentifierHandler::Scan(SourceCodeWrapper& wrapper) { } if (s[0] == '#') { - throw LexerError(std::string("Not a keyword, started with #: ") + s); + return std::unexpected(LexerError(std::string("Not a keyword, started with #: ") + s)); } return std::make_optional(TokenFactory::MakeIdent(std::move(s), wrapper.GetLine(), wrapper.GetTokenCol())); } + +} // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/IdentifierHandler.hpp b/lib/lexer/handlers/IdentifierHandler.hpp index fbabc98..45f9b8c 100644 --- a/lib/lexer/handlers/IdentifierHandler.hpp +++ b/lib/lexer/handlers/IdentifierHandler.hpp @@ -1,11 +1,15 @@ -#ifndef IDENTIFIERHANDLER_HPP_ -#define IDENTIFIERHANDLER_HPP_ +#ifndef LEXER_IDENTIFIERHANDLER_HPP_ +#define LEXER_IDENTIFIERHANDLER_HPP_ #include "Handler.hpp" +namespace ovum::compiler::lexer { + class IdentifierHandler : public Handler { public: - OptToken Scan(SourceCodeWrapper& wrapper) override; + std::expected Scan(SourceCodeWrapper& wrapper) override; }; -#endif // IDENTIFIERHANDLER_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_IDENTIFIERHANDLER_HPP_ diff --git a/lib/lexer/handlers/NewlineHandler.cpp b/lib/lexer/handlers/NewlineHandler.cpp index de5b775..9def50e 100644 --- a/lib/lexer/handlers/NewlineHandler.cpp +++ b/lib/lexer/handlers/NewlineHandler.cpp @@ -1,7 +1,11 @@ #include "NewlineHandler.hpp" -#include "lib/lexer/tokens/TokenFactory.hpp" +#include -OptToken NewlineHandler::Scan(SourceCodeWrapper& wrapper) { +namespace ovum::compiler::lexer { + +std::expected NewlineHandler::Scan(SourceCodeWrapper& wrapper) { return std::make_optional(TokenFactory::MakeNewline(wrapper.GetLine() - 1, wrapper.GetTokenCol())); } + +} // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/NewlineHandler.hpp b/lib/lexer/handlers/NewlineHandler.hpp index 42cd181..b85a71c 100644 --- a/lib/lexer/handlers/NewlineHandler.hpp +++ b/lib/lexer/handlers/NewlineHandler.hpp @@ -1,11 +1,15 @@ -#ifndef NEWLINEHANDLER_HPP_ -#define NEWLINEHANDLER_HPP_ +#ifndef LEXER_NEWLINEHANDLER_HPP_ +#define LEXER_NEWLINEHANDLER_HPP_ #include "Handler.hpp" +namespace ovum::compiler::lexer { + class NewlineHandler : public Handler { public: - OptToken Scan(SourceCodeWrapper& wrapper) override; + std::expected Scan(SourceCodeWrapper& wrapper) override; }; -#endif // NEWLINEHANDLER_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_NEWLINEHANDLER_HPP_ diff --git a/lib/lexer/handlers/NumberHandler.cpp b/lib/lexer/handlers/NumberHandler.cpp index 9c30220..9d59c6e 100644 --- a/lib/lexer/handlers/NumberHandler.cpp +++ b/lib/lexer/handlers/NumberHandler.cpp @@ -3,8 +3,11 @@ #include #include +#include + #include "lib/lexer/LexerError.hpp" -#include "tokens/TokenFactory.hpp" + +namespace ovum::compiler::lexer { namespace { inline constexpr int kHexAlphaOffset = 10; @@ -29,7 +32,7 @@ inline bool IsIdentStart(char c) noexcept { return std::isalpha(u) != 0 || u == '_'; } -inline bool ParseExponent(SourceCodeWrapper& w, std::string& raw) { +inline std::expected ParseExponent(SourceCodeWrapper& w, std::string& raw) { if (w.Peek() == 'e' || w.Peek() == 'E') { raw.push_back(w.Advance()); @@ -38,7 +41,7 @@ inline bool ParseExponent(SourceCodeWrapper& w, std::string& raw) { } if (!IsDec(w.Peek())) { - throw LexerError("Malformed exponent"); + return std::unexpected(LexerError("Malformed exponent")); } w.ConsumeWhile(raw, IsDec); @@ -48,35 +51,37 @@ inline bool ParseExponent(SourceCodeWrapper& w, std::string& raw) { return false; } -inline double ParseDoubleStrict(const std::string& raw) { +inline std::expected ParseDoubleStrict(const std::string& raw) { try { return std::stod(raw); } catch (...) { - throw LexerError(std::string("Malformed float literal: ") + raw); + return std::unexpected(LexerError(std::string("Malformed float literal: ") + raw)); } } -inline long long ParseDecIntStrict(const std::string& raw) { +inline std::expected ParseDecIntStrict(const std::string& raw) { try { return std::stoll(raw); } catch (...) { - throw LexerError(std::string("Malformed integer literal: ") + raw); + return std::unexpected(LexerError(std::string("Malformed integer literal: ") + raw)); } } -inline void EnsureNoIdentTail(SourceCodeWrapper& w, const char* ctx) { +inline std::expected EnsureNoIdentTail(SourceCodeWrapper& w, const char* ctx) { if (IsIdentStart(w.Peek())) { - throw LexerError(std::string("Unexpected identifier after ") + ctx); + return std::unexpected(LexerError(std::string("Unexpected identifier after ") + ctx)); } + return {}; } -inline void EnsureNoSecondDotWithDigits(SourceCodeWrapper& w) { +inline std::expected EnsureNoSecondDotWithDigits(SourceCodeWrapper& w) { if (w.Peek() == '.' && IsDec(w.Peek(1))) { - throw LexerError("Malformed float literal: duplicate decimal point"); + return std::unexpected(LexerError("Malformed float literal: duplicate decimal point")); } + return {}; } -OptToken NumberHandler::Scan(SourceCodeWrapper& wrapper) { +std::expected NumberHandler::Scan(SourceCodeWrapper& wrapper) { std::string raw; const char first = wrapper.CurrentChar(); @@ -88,12 +93,24 @@ OptToken NumberHandler::Scan(SourceCodeWrapper& wrapper) { } wrapper.ConsumeWhile(raw, IsDec); - ParseExponent(wrapper, raw); - EnsureNoSecondDotWithDigits(wrapper); - EnsureNoIdentTail(wrapper, "number"); - const double v = ParseDoubleStrict(raw); + auto exp_result = ParseExponent(wrapper, raw); + if (!exp_result) { + return std::unexpected(exp_result.error()); + } + auto dot_result = EnsureNoSecondDotWithDigits(wrapper); + if (!dot_result) { + return std::unexpected(dot_result.error()); + } + auto ident_result = EnsureNoIdentTail(wrapper, "number"); + if (!ident_result) { + return std::unexpected(ident_result.error()); + } + auto v_result = ParseDoubleStrict(raw); + if (!v_result) { + return std::unexpected(v_result.error()); + } - return TokenFactory::MakeFloatLiteral(std::move(raw), v, wrapper.GetLine(), wrapper.GetTokenCol()); + return TokenFactory::MakeFloatLiteral(std::move(raw), v_result.value(), wrapper.GetLine(), wrapper.GetTokenCol()); } wrapper.RetreatOne(); @@ -103,16 +120,19 @@ OptToken NumberHandler::Scan(SourceCodeWrapper& wrapper) { raw.push_back(wrapper.Advance()); if (!IsHex(wrapper.Peek())) { - throw LexerError("Malformed hex literal: expected hex digit after 0x"); + return std::unexpected(LexerError("Malformed hex literal: expected hex digit after 0x")); } wrapper.ConsumeWhile(raw, IsHex); if (wrapper.Peek() == '.') { - throw LexerError("Hex literal cannot have decimal point"); + return std::unexpected(LexerError("Hex literal cannot have decimal point")); } - EnsureNoIdentTail(wrapper, "hex literal"); + auto ident_result = EnsureNoIdentTail(wrapper, "hex literal"); + if (!ident_result) { + return std::unexpected(ident_result.error()); + } long long val = 0; @@ -137,16 +157,19 @@ OptToken NumberHandler::Scan(SourceCodeWrapper& wrapper) { raw.push_back(wrapper.Advance()); if (!IsBin(wrapper.Peek())) { - throw LexerError("Malformed binary literal: expected binary digit after 0b"); + return std::unexpected(LexerError("Malformed binary literal: expected binary digit after 0b")); } wrapper.ConsumeWhile(raw, IsBin); if (wrapper.Peek() == '.') { - throw LexerError("Binary literal cannot have decimal point"); + return std::unexpected(LexerError("Binary literal cannot have decimal point")); } - EnsureNoIdentTail(wrapper, "binary literal"); + auto ident_result = EnsureNoIdentTail(wrapper, "binary literal"); + if (!ident_result) { + return std::unexpected(ident_result.error()); + } long long val = 0; for (size_t i = 2; i < raw.size(); ++i) { @@ -165,27 +188,57 @@ OptToken NumberHandler::Scan(SourceCodeWrapper& wrapper) { wrapper.ConsumeWhile(raw, IsDec); } - ParseExponent(wrapper, raw); - EnsureNoSecondDotWithDigits(wrapper); - EnsureNoIdentTail(wrapper, "number"); - const double v = ParseDoubleStrict(raw); - return TokenFactory::MakeFloatLiteral(std::move(raw), v, wrapper.GetLine(), wrapper.GetTokenCol()); + auto exp_result = ParseExponent(wrapper, raw); + if (!exp_result) { + return std::unexpected(exp_result.error()); + } + auto dot_result = EnsureNoSecondDotWithDigits(wrapper); + if (!dot_result) { + return std::unexpected(dot_result.error()); + } + auto ident_result = EnsureNoIdentTail(wrapper, "number"); + if (!ident_result) { + return std::unexpected(ident_result.error()); + } + auto v_result = ParseDoubleStrict(raw); + if (!v_result) { + return std::unexpected(v_result.error()); + } + return TokenFactory::MakeFloatLiteral(std::move(raw), v_result.value(), wrapper.GetLine(), wrapper.GetTokenCol()); } - const bool had_exp = ParseExponent(wrapper, raw); + auto exp_result = ParseExponent(wrapper, raw); + if (!exp_result) { + return std::unexpected(exp_result.error()); + } + const bool had_exp = exp_result.value(); if (had_exp) { - EnsureNoIdentTail(wrapper, "number"); - const double v = ParseDoubleStrict(raw); - return TokenFactory::MakeFloatLiteral(std::move(raw), v, wrapper.GetLine(), wrapper.GetTokenCol()); + auto ident_result = EnsureNoIdentTail(wrapper, "number"); + if (!ident_result) { + return std::unexpected(ident_result.error()); + } + auto v_result = ParseDoubleStrict(raw); + if (!v_result) { + return std::unexpected(v_result.error()); + } + return TokenFactory::MakeFloatLiteral(std::move(raw), v_result.value(), wrapper.GetLine(), wrapper.GetTokenCol()); } if (!raw.empty() && (raw.back() == 'e' || raw.back() == 'E' || raw.back() == '+' || raw.back() == '-')) { - throw LexerError(std::string("Malformed float literal: ") + raw); + return std::unexpected(LexerError(std::string("Malformed float literal: ") + raw)); } - EnsureNoIdentTail(wrapper, "integer literal"); - const long long vi = ParseDecIntStrict(raw); + auto ident_result = EnsureNoIdentTail(wrapper, "integer literal"); + if (!ident_result) { + return std::unexpected(ident_result.error()); + } + auto vi_result = ParseDecIntStrict(raw); + if (!vi_result) { + return std::unexpected(vi_result.error()); + } - return TokenFactory::MakeIntLiteral(std::move(raw), vi, wrapper.GetLine(), wrapper.GetTokenCol()); + return TokenFactory::MakeIntLiteral(std::move(raw), vi_result.value(), wrapper.GetLine(), wrapper.GetTokenCol()); } + +} // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/NumberHandler.hpp b/lib/lexer/handlers/NumberHandler.hpp index 2dfcad9..10cef26 100644 --- a/lib/lexer/handlers/NumberHandler.hpp +++ b/lib/lexer/handlers/NumberHandler.hpp @@ -1,11 +1,15 @@ -#ifndef NUMBERHANDLER_HPP_ -#define NUMBERHANDLER_HPP_ +#ifndef LEXER_NUMBERHANDLER_HPP_ +#define LEXER_NUMBERHANDLER_HPP_ #include "Handler.hpp" +namespace ovum::compiler::lexer { + class NumberHandler : public Handler { public: - OptToken Scan(SourceCodeWrapper& wrapper) override; + std::expected Scan(SourceCodeWrapper& wrapper) override; }; -#endif // NUMBERHANDLER_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_NUMBERHANDLER_HPP_ diff --git a/lib/lexer/handlers/OperatorHandler.cpp b/lib/lexer/handlers/OperatorHandler.cpp index 4c57f4b..82c9554 100644 --- a/lib/lexer/handlers/OperatorHandler.cpp +++ b/lib/lexer/handlers/OperatorHandler.cpp @@ -1,8 +1,10 @@ #include "OperatorHandler.hpp" -#include "lib/lexer/tokens/TokenFactory.hpp" +#include -OptToken OperatorHandler::Scan(SourceCodeWrapper& wrapper) { +namespace ovum::compiler::lexer { + +std::expected OperatorHandler::Scan(SourceCodeWrapper& wrapper) { std::string op; op.push_back(wrapper.CurrentChar()); char p = wrapper.Peek(); @@ -18,3 +20,5 @@ OptToken OperatorHandler::Scan(SourceCodeWrapper& wrapper) { return std::make_optional(TokenFactory::MakeOperator(std::move(op), wrapper.GetLine(), wrapper.GetTokenCol())); } + +} // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/OperatorHandler.hpp b/lib/lexer/handlers/OperatorHandler.hpp index 1185a8f..2603c5f 100644 --- a/lib/lexer/handlers/OperatorHandler.hpp +++ b/lib/lexer/handlers/OperatorHandler.hpp @@ -1,11 +1,15 @@ -#ifndef OPERATORHANDLER_HPP_ -#define OPERATORHANDLER_HPP_ +#ifndef LEXER_OPERATORHANDLER_HPP_ +#define LEXER_OPERATORHANDLER_HPP_ #include "Handler.hpp" +namespace ovum::compiler::lexer { + class OperatorHandler : public Handler { public: - OptToken Scan(SourceCodeWrapper& wrapper) override; + std::expected Scan(SourceCodeWrapper& wrapper) override; }; -#endif // OPERATORHANDLER_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_OPERATORHANDLER_HPP_ diff --git a/lib/lexer/handlers/PunctHandler.cpp b/lib/lexer/handlers/PunctHandler.cpp index 28bc6f8..09eefea 100644 --- a/lib/lexer/handlers/PunctHandler.cpp +++ b/lib/lexer/handlers/PunctHandler.cpp @@ -1,8 +1,12 @@ #include "PunctHandler.hpp" -#include "lib/lexer/tokens/TokenFactory.hpp" +#include -OptToken PunctHandler::Scan(SourceCodeWrapper& wrapper) { +namespace ovum::compiler::lexer { + +std::expected PunctHandler::Scan(SourceCodeWrapper& wrapper) { char c = wrapper.CurrentChar(); return std::make_optional(TokenFactory::MakePunct(c, wrapper.GetLine(), wrapper.GetTokenCol())); } + +} // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/PunctHandler.hpp b/lib/lexer/handlers/PunctHandler.hpp index d69e5a8..baf4bc8 100644 --- a/lib/lexer/handlers/PunctHandler.hpp +++ b/lib/lexer/handlers/PunctHandler.hpp @@ -1,11 +1,15 @@ -#ifndef PUNCTHANDLER_HPP_ -#define PUNCTHANDLER_HPP_ +#ifndef LEXER_PUNCTHANDLER_HPP_ +#define LEXER_PUNCTHANDLER_HPP_ #include "Handler.hpp" +namespace ovum::compiler::lexer { + class PunctHandler : public Handler { public: - OptToken Scan(SourceCodeWrapper& wrapper) override; + std::expected Scan(SourceCodeWrapper& wrapper) override; }; -#endif // PUNCTHANDLER_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_PUNCTHANDLER_HPP_ diff --git a/lib/lexer/handlers/SlashHandler.cpp b/lib/lexer/handlers/SlashHandler.cpp index 035645e..6fecd1d 100644 --- a/lib/lexer/handlers/SlashHandler.cpp +++ b/lib/lexer/handlers/SlashHandler.cpp @@ -1,9 +1,12 @@ #include "SlashHandler.hpp" +#include + #include "lib/lexer/LexerError.hpp" -#include "tokens/TokenFactory.hpp" -OptToken SlashHandler::Scan(SourceCodeWrapper& wrapper) { +namespace ovum::compiler::lexer { + +std::expected SlashHandler::Scan(SourceCodeWrapper& wrapper) { if (wrapper.Peek() == '/') { std::string txt; @@ -38,7 +41,7 @@ OptToken SlashHandler::Scan(SourceCodeWrapper& wrapper) { } if (!closed) { - throw LexerError("Unterminated block comment"); + return std::unexpected(LexerError("Unterminated block comment")); } if (wrapper.IsKeepComments()) { @@ -50,3 +53,5 @@ OptToken SlashHandler::Scan(SourceCodeWrapper& wrapper) { return std::make_optional(TokenFactory::MakeOperator(std::string(1, '/'), wrapper.GetLine(), wrapper.GetTokenCol())); } + +} // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/SlashHandler.hpp b/lib/lexer/handlers/SlashHandler.hpp index 369c5a3..5719c95 100644 --- a/lib/lexer/handlers/SlashHandler.hpp +++ b/lib/lexer/handlers/SlashHandler.hpp @@ -1,11 +1,15 @@ -#ifndef SLASHHANDLER_HPP_ -#define SLASHHANDLER_HPP_ +#ifndef LEXER_SLASHHANDLER_HPP_ +#define LEXER_SLASHHANDLER_HPP_ #include "Handler.hpp" +namespace ovum::compiler::lexer { + class SlashHandler : public Handler { public: - OptToken Scan(SourceCodeWrapper& wrapper) override; + std::expected Scan(SourceCodeWrapper& wrapper) override; }; -#endif // SLASHHANDLER_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_SLASHHANDLER_HPP_ diff --git a/lib/lexer/handlers/StringHandler.cpp b/lib/lexer/handlers/StringHandler.cpp index c1eb07c..f1c31dd 100644 --- a/lib/lexer/handlers/StringHandler.cpp +++ b/lib/lexer/handlers/StringHandler.cpp @@ -2,10 +2,13 @@ #include +#include + #include "lib/lexer/LexerError.hpp" -#include "tokens/TokenFactory.hpp" -OptToken StringHandler::Scan(SourceCodeWrapper& wrapper) { +namespace ovum::compiler::lexer { + +std::expected StringHandler::Scan(SourceCodeWrapper& wrapper) { std::string raw; std::string out; raw.push_back('"'); @@ -21,7 +24,7 @@ OptToken StringHandler::Scan(SourceCodeWrapper& wrapper) { if (c == '\\') { if (wrapper.IsAtEnd()) { - throw LexerError("Unterminated string literal (backslash at EOF)"); + return std::unexpected(LexerError("Unterminated string literal (backslash at EOF)")); } char e = wrapper.Advance(); @@ -47,16 +50,18 @@ OptToken StringHandler::Scan(SourceCodeWrapper& wrapper) { out.push_back('\0'); break; default: - throw LexerError(std::string("Unknown escape in string literal: \\") + e); + return std::unexpected(LexerError(std::string("Unknown escape in string literal: \\") + e)); } } else { if (c == '\n') { - throw LexerError("Unterminated string literal (newline inside)"); + return std::unexpected(LexerError("Unterminated string literal (newline inside)")); } out.push_back(c); } } - throw LexerError("Unterminated string literal (EOF reached)"); + return std::unexpected(LexerError("Unterminated string literal (EOF reached)")); } + +} // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/StringHandler.hpp b/lib/lexer/handlers/StringHandler.hpp index a735134..929c8d4 100644 --- a/lib/lexer/handlers/StringHandler.hpp +++ b/lib/lexer/handlers/StringHandler.hpp @@ -1,11 +1,15 @@ -#ifndef STRINGHANDLER_H -#define STRINGHANDLER_H +#ifndef LEXER_STRINGHANDLER_HPP_ +#define LEXER_STRINGHANDLER_HPP_ #include "Handler.hpp" +namespace ovum::compiler::lexer { + class StringHandler : public Handler { public: - OptToken Scan(SourceCodeWrapper& wrapper) override; + std::expected Scan(SourceCodeWrapper& wrapper) override; }; -#endif // STRINGHANDLER_H +} // namespace ovum::compiler::lexer + +#endif // LEXER_STRINGHANDLER_HPP_ diff --git a/lib/lexer/handlers/WhitespaceHandler.cpp b/lib/lexer/handlers/WhitespaceHandler.cpp index d2f1d1d..060cb0b 100644 --- a/lib/lexer/handlers/WhitespaceHandler.cpp +++ b/lib/lexer/handlers/WhitespaceHandler.cpp @@ -1,6 +1,8 @@ #include "WhitespaceHandler.hpp" -OptToken WhitespaceHandler::Scan(SourceCodeWrapper& wrapper) { +namespace ovum::compiler::lexer { + +std::expected WhitespaceHandler::Scan(SourceCodeWrapper& wrapper) { while (!wrapper.IsAtEnd()) { if (char p = wrapper.Peek(); p != ' ' && p != '\t' && p != '\r') { break; @@ -11,3 +13,5 @@ OptToken WhitespaceHandler::Scan(SourceCodeWrapper& wrapper) { return std::nullopt; } + +} // namespace ovum::compiler::lexer diff --git a/lib/lexer/handlers/WhitespaceHandler.hpp b/lib/lexer/handlers/WhitespaceHandler.hpp index 3bab9db..f9f7a29 100644 --- a/lib/lexer/handlers/WhitespaceHandler.hpp +++ b/lib/lexer/handlers/WhitespaceHandler.hpp @@ -1,11 +1,15 @@ -#ifndef WHITESPACEHANDLER_HPP_ -#define WHITESPACEHANDLER_HPP_ +#ifndef LEXER_WHITESPACEHANDLER_HPP_ +#define LEXER_WHITESPACEHANDLER_HPP_ #include "Handler.hpp" +namespace ovum::compiler::lexer { + class WhitespaceHandler : public Handler { public: - OptToken Scan(SourceCodeWrapper& wrapper) override; + std::expected Scan(SourceCodeWrapper& wrapper) override; }; -#endif // WHITESPACEHANDLER_HPP_ +} // namespace ovum::compiler::lexer + +#endif // LEXER_WHITESPACEHANDLER_HPP_ diff --git a/lib/lexer/tokens/CommentToken.cpp b/lib/lexer/tokens/CommentToken.cpp deleted file mode 100644 index 27397b1..0000000 --- a/lib/lexer/tokens/CommentToken.cpp +++ /dev/null @@ -1,33 +0,0 @@ -#include "CommentToken.hpp" - -#include - -CommentToken::CommentToken(std::string txt, const TokenPosition& position) : - text_(std::move(txt)), position_(position) { -} - -std::string CommentToken::GetStringType() const noexcept { - return "COMMENT"; -} - -std::string CommentToken::GetLexeme() const noexcept { - return text_; -} - -TokenPtr CommentToken::Clone() const { - return std::make_shared(*this); -} - -void CommentToken::Accept(TokenVisitor& visitor) const { - visitor.Visit(*this); -} - -std::string CommentToken::ToString() const { - std::ostringstream os; - os << "Token(COMMENT, '" << text_ << "', @" << position_.GetLine() << ":" << position_.GetColumn() << ")"; - return os.str(); -} - -const TokenPosition& CommentToken::GetPosition() const noexcept { - return position_; -} diff --git a/lib/lexer/tokens/CommentToken.hpp b/lib/lexer/tokens/CommentToken.hpp deleted file mode 100644 index 319da04..0000000 --- a/lib/lexer/tokens/CommentToken.hpp +++ /dev/null @@ -1,30 +0,0 @@ -#ifndef COMMENTTOKEN_HPP_ -#define COMMENTTOKEN_HPP_ - -#include - -#include "Token.hpp" -#include "TokenVisitor.hpp" - -class CommentToken final : public Token { -public: - CommentToken(std::string txt, const TokenPosition& position); - - [[nodiscard]] std::string GetStringType() const noexcept override; - - [[nodiscard]] std::string GetLexeme() const noexcept override; - - [[nodiscard]] TokenPtr Clone() const override; - - void Accept(TokenVisitor& visitor) const override; - - [[nodiscard]] std::string ToString() const override; - - [[nodiscard]] const TokenPosition& GetPosition() const noexcept override; - -private: - std::string text_; - TokenPosition position_; -}; - -#endif // COMMENTTOKEN_HPP_ diff --git a/lib/lexer/tokens/EofToken.cpp b/lib/lexer/tokens/EofToken.cpp deleted file mode 100644 index de3ad16..0000000 --- a/lib/lexer/tokens/EofToken.cpp +++ /dev/null @@ -1,32 +0,0 @@ -#include "EofToken.hpp" - -#include - -EofToken::EofToken(const TokenPosition& position) : position_(position) { -} - -std::string EofToken::GetStringType() const noexcept { - return "EOF"; -} - -std::string EofToken::GetLexeme() const noexcept { - return ""; -} - -TokenPtr EofToken::Clone() const { - return std::make_shared(*this); -} - -void EofToken::Accept(TokenVisitor& visitor) const { - visitor.Visit(*this); -} - -std::string EofToken::ToString() const { - std::ostringstream os; - os << "Token(EOF, @" << position_.GetLine() << ":" << position_.GetColumn() << ")"; - return os.str(); -} - -const TokenPosition& EofToken::GetPosition() const noexcept { - return position_; -} diff --git a/lib/lexer/tokens/EofToken.hpp b/lib/lexer/tokens/EofToken.hpp deleted file mode 100644 index d6cefe7..0000000 --- a/lib/lexer/tokens/EofToken.hpp +++ /dev/null @@ -1,29 +0,0 @@ -#ifndef EOFTOKEN_HPP_ -#define EOFTOKEN_HPP_ - -#include - -#include "Token.hpp" -#include "TokenVisitor.hpp" - -class EofToken final : public Token { -public: - EofToken(const TokenPosition& position); - - [[nodiscard]] std::string GetStringType() const noexcept override; - - [[nodiscard]] std::string GetLexeme() const noexcept override; - - [[nodiscard]] TokenPtr Clone() const override; - - void Accept(TokenVisitor& visitor) const override; - - [[nodiscard]] std::string ToString() const override; - - [[nodiscard]] const TokenPosition& GetPosition() const noexcept override; - -private: - TokenPosition position_; -}; - -#endif // EOFTOKEN_HPP_ diff --git a/lib/lexer/tokens/IdentToken.cpp b/lib/lexer/tokens/IdentToken.cpp deleted file mode 100644 index 892eff3..0000000 --- a/lib/lexer/tokens/IdentToken.cpp +++ /dev/null @@ -1,36 +0,0 @@ -#include "IdentToken.hpp" - -#include - -IdentToken::IdentToken(std::string lex, const TokenPosition& position) : lexeme_(std::move(lex)), position_(position) { -} - -std::string IdentToken::GetStringType() const noexcept { - return "IDENT"; -} - -std::string IdentToken::GetLexeme() const noexcept { - return lexeme_; -} - -TokenPtr IdentToken::Clone() const { - return std::make_shared(*this); -} - -void IdentToken::Accept(TokenVisitor& visitor) const { - visitor.Visit(*this); -} - -std::string IdentToken::ToString() const { - std::ostringstream os; - os << "Token(IDENT, '" << lexeme_ << "', @" << position_.GetLine() << ":" << position_.GetColumn() << ")"; - return os.str(); -} - -const TokenPosition& IdentToken::GetPosition() const noexcept { - return position_; -} - -const std::string& IdentToken::GetName() const noexcept { - return lexeme_; -} diff --git a/lib/lexer/tokens/IdentToken.hpp b/lib/lexer/tokens/IdentToken.hpp deleted file mode 100644 index e3709ff..0000000 --- a/lib/lexer/tokens/IdentToken.hpp +++ /dev/null @@ -1,32 +0,0 @@ -#ifndef IDENTTOKEN_HPP_ -#define IDENTTOKEN_HPP_ - -#include - -#include "Token.hpp" -#include "TokenVisitor.hpp" - -class IdentToken final : public Token { -public: - IdentToken(std::string lex, const TokenPosition& position); - - [[nodiscard]] std::string GetStringType() const noexcept override; - - [[nodiscard]] std::string GetLexeme() const noexcept override; - - [[nodiscard]] TokenPtr Clone() const override; - - void Accept(TokenVisitor& visitor) const override; - - [[nodiscard]] std::string ToString() const override; - - [[nodiscard]] const std::string& GetName() const noexcept; - - [[nodiscard]] const TokenPosition& GetPosition() const noexcept override; - -private: - std::string lexeme_; - TokenPosition position_; -}; - -#endif // IDENTTOKEN_HPP_ diff --git a/lib/lexer/tokens/KeywordToken.cpp b/lib/lexer/tokens/KeywordToken.cpp deleted file mode 100644 index 82cd8d1..0000000 --- a/lib/lexer/tokens/KeywordToken.cpp +++ /dev/null @@ -1,33 +0,0 @@ -#include "KeywordToken.hpp" - -#include - -KeywordToken::KeywordToken(std::string lex, const TokenPosition& position) : - lexeme_(std::move(lex)), position_(position) { -} - -std::string KeywordToken::GetStringType() const noexcept { - return "KEYWORD"; -} - -std::string KeywordToken::GetLexeme() const noexcept { - return lexeme_; -} - -TokenPtr KeywordToken::Clone() const { - return std::make_shared(*this); -} - -void KeywordToken::Accept(TokenVisitor& visitor) const { - visitor.Visit(*this); -} - -std::string KeywordToken::ToString() const { - std::ostringstream os; - os << "Token(KEYWORD, '" << lexeme_ << "', @" << position_.GetLine() << ":" << position_.GetColumn() << ")"; - return os.str(); -} - -const TokenPosition& KeywordToken::GetPosition() const noexcept { - return position_; -} diff --git a/lib/lexer/tokens/KeywordToken.hpp b/lib/lexer/tokens/KeywordToken.hpp deleted file mode 100644 index 7860e5f..0000000 --- a/lib/lexer/tokens/KeywordToken.hpp +++ /dev/null @@ -1,30 +0,0 @@ -#ifndef KEYWORDTOKEN_HPP_ -#define KEYWORDTOKEN_HPP_ - -#include - -#include "Token.hpp" -#include "TokenVisitor.hpp" - -class KeywordToken final : public Token { -public: - KeywordToken(std::string lex, const TokenPosition& position); - - [[nodiscard]] std::string GetStringType() const noexcept override; - - [[nodiscard]] std::string GetLexeme() const noexcept override; - - [[nodiscard]] TokenPtr Clone() const override; - - void Accept(TokenVisitor& visitor) const override; - - [[nodiscard]] std::string ToString() const override; - - [[nodiscard]] const TokenPosition& GetPosition() const noexcept override; - -private: - std::string lexeme_; - TokenPosition position_; -}; - -#endif // KEYWORDTOKEN_HPP_ diff --git a/lib/lexer/tokens/LiteralToken.cpp b/lib/lexer/tokens/LiteralToken.cpp deleted file mode 100644 index a18c863..0000000 --- a/lib/lexer/tokens/LiteralToken.cpp +++ /dev/null @@ -1,49 +0,0 @@ -#include "LiteralToken.hpp" - -#include -#include - -LiteralToken::LiteralToken(std::string rawLexeme, std::unique_ptr value, const TokenPosition& position) : - lexeme_(std::move(rawLexeme)), value_(std::move(value)), position_(position) { -} - -std::string LiteralToken::GetStringType() const noexcept { - return "LITERAL:" + value_->GetTypeName(); -} - -std::string LiteralToken::GetLexeme() const noexcept { - return lexeme_; -} - -const Value* LiteralToken::GetValue() const noexcept { - return value_.get(); -} - -Value* LiteralToken::GetValue() noexcept { - return value_.get(); -} - -TokenPtr LiteralToken::Clone() const { - std::unique_ptr vcopy = value_ ? value_->Clone() : nullptr; - return std::make_shared(lexeme_, std::move(vcopy), position_); -} - -void LiteralToken::Accept(TokenVisitor& visitor) const { - visitor.Visit(*this); -} - -std::string LiteralToken::ToString() const { - std::ostringstream os; - os << "Token(" << GetStringType() << ", '" << lexeme_ << "'"; - - if (value_) { - os << ", " << value_->ToString(); - } - - os << ", @" << position_.GetLine() << ":" << position_.GetColumn() << ")"; - return os.str(); -} - -const TokenPosition& LiteralToken::GetPosition() const noexcept { - return position_; -} diff --git a/lib/lexer/tokens/LiteralToken.hpp b/lib/lexer/tokens/LiteralToken.hpp deleted file mode 100644 index 2b30d17..0000000 --- a/lib/lexer/tokens/LiteralToken.hpp +++ /dev/null @@ -1,37 +0,0 @@ -#ifndef LITERALTOKEN_HPP_ -#define LITERALTOKEN_HPP_ - -#include -#include - -#include "Token.hpp" -#include "TokenVisitor.hpp" -#include "lib/lexer/values/Value.hpp" - -class LiteralToken final : public Token { -public: - LiteralToken(std::string rawLexeme, std::unique_ptr value, const TokenPosition& position); - - [[nodiscard]] std::string GetStringType() const noexcept override; - - [[nodiscard]] std::string GetLexeme() const noexcept override; - - [[nodiscard]] const Value* GetValue() const noexcept; - - Value* GetValue() noexcept; - - [[nodiscard]] TokenPtr Clone() const override; - - void Accept(TokenVisitor& visitor) const override; - - [[nodiscard]] std::string ToString() const override; - - [[nodiscard]] const TokenPosition& GetPosition() const noexcept override; - -private: - std::string lexeme_; - std::unique_ptr value_; - TokenPosition position_; -}; - -#endif // LITERALTOKEN_HPP_ diff --git a/lib/lexer/tokens/NewlineToken.cpp b/lib/lexer/tokens/NewlineToken.cpp deleted file mode 100644 index bcbdcaa..0000000 --- a/lib/lexer/tokens/NewlineToken.cpp +++ /dev/null @@ -1,32 +0,0 @@ -#include "NewlineToken.hpp" - -#include - -NewlineToken::NewlineToken(const TokenPosition& position) : lexeme_("\\n"), position_(position) { -} - -std::string NewlineToken::GetStringType() const noexcept { - return "NEWLINE"; -} - -std::string NewlineToken::GetLexeme() const noexcept { - return lexeme_; -} - -TokenPtr NewlineToken::Clone() const { - return std::make_shared(*this); -} - -void NewlineToken::Accept(TokenVisitor& visitor) const { - visitor.Visit(*this); -} - -std::string NewlineToken::ToString() const { - std::ostringstream os; - os << "Token(NEWLINE, '\\n', @" << position_.GetLine() << ":" << position_.GetColumn() << ")"; - return os.str(); -} - -const TokenPosition& NewlineToken::GetPosition() const noexcept { - return position_; -} diff --git a/lib/lexer/tokens/NewlineToken.hpp b/lib/lexer/tokens/NewlineToken.hpp deleted file mode 100644 index 02686b4..0000000 --- a/lib/lexer/tokens/NewlineToken.hpp +++ /dev/null @@ -1,30 +0,0 @@ -#ifndef NEWLINETOKEN_HPP_ -#define NEWLINETOKEN_HPP_ - -#include - -#include "Token.hpp" -#include "TokenVisitor.hpp" - -class NewlineToken final : public Token { -public: - NewlineToken(const TokenPosition& position); - - [[nodiscard]] std::string GetStringType() const noexcept override; - - [[nodiscard]] std::string GetLexeme() const noexcept override; - - [[nodiscard]] TokenPtr Clone() const override; - - void Accept(TokenVisitor& visitor) const override; - - [[nodiscard]] std::string ToString() const override; - - [[nodiscard]] const TokenPosition& GetPosition() const noexcept override; - -private: - std::string lexeme_; - TokenPosition position_; -}; - -#endif // NEWLINETOKEN_HPP_ diff --git a/lib/lexer/tokens/OperatorToken.cpp b/lib/lexer/tokens/OperatorToken.cpp deleted file mode 100644 index 32be80d..0000000 --- a/lib/lexer/tokens/OperatorToken.cpp +++ /dev/null @@ -1,33 +0,0 @@ -#include "OperatorToken.hpp" - -#include - -OperatorToken::OperatorToken(std::string op, const TokenPosition& position) : - lexeme_(std::move(op)), position_(position) { -} - -std::string OperatorToken::GetStringType() const noexcept { - return "OPERATOR"; -} - -std::string OperatorToken::GetLexeme() const noexcept { - return lexeme_; -} - -TokenPtr OperatorToken::Clone() const { - return std::make_shared(*this); -} - -void OperatorToken::Accept(TokenVisitor& visitor) const { - visitor.Visit(*this); -} - -std::string OperatorToken::ToString() const { - std::ostringstream os; - os << "Token(OPERATOR, '" << lexeme_ << "', @" << position_.GetLine() << ":" << position_.GetColumn() << ")"; - return os.str(); -} - -const TokenPosition& OperatorToken::GetPosition() const noexcept { - return position_; -} diff --git a/lib/lexer/tokens/OperatorToken.hpp b/lib/lexer/tokens/OperatorToken.hpp deleted file mode 100644 index 5f82a6b..0000000 --- a/lib/lexer/tokens/OperatorToken.hpp +++ /dev/null @@ -1,31 +0,0 @@ -#ifndef OPERATORTOKEN_HPP_ -#define OPERATORTOKEN_HPP_ - -#include - -#include "Token.hpp" - -class TokenVisitor; - -class OperatorToken final : public Token { -public: - OperatorToken(std::string op, const TokenPosition& position); - - [[nodiscard]] std::string GetStringType() const noexcept override; - - [[nodiscard]] std::string GetLexeme() const noexcept override; - - [[nodiscard]] TokenPtr Clone() const override; - - void Accept(TokenVisitor& visitor) const override; - - [[nodiscard]] std::string ToString() const override; - - [[nodiscard]] const TokenPosition& GetPosition() const noexcept override; - -private: - std::string lexeme_; - TokenPosition position_; -}; - -#endif // OPERATORTOKEN_HPP_ diff --git a/lib/lexer/tokens/PunctToken.cpp b/lib/lexer/tokens/PunctToken.cpp deleted file mode 100644 index 47348cc..0000000 --- a/lib/lexer/tokens/PunctToken.cpp +++ /dev/null @@ -1,35 +0,0 @@ -#include -#include "PunctToken.hpp" - -PunctToken::PunctToken(char ch, const TokenPosition& position) : lexeme_(1, ch), position_(position) { -} - -PunctToken::PunctToken(std::string punct, const TokenPosition& position) : - lexeme_(std::move(punct)), position_(position) { -} - -std::string PunctToken::GetStringType() const noexcept { - return "PUNCT"; -} - -std::string PunctToken::GetLexeme() const noexcept { - return lexeme_; -} - -TokenPtr PunctToken::Clone() const { - return std::make_shared(*this); -} - -void PunctToken::Accept(TokenVisitor& visitor) const { - visitor.Visit(*this); -} - -std::string PunctToken::ToString() const { - std::ostringstream os; - os << "Token(PUNCT, '" << lexeme_ << "', @" << position_.GetLine() << ":" << position_.GetColumn() << ")"; - return os.str(); -} - -const TokenPosition& PunctToken::GetPosition() const noexcept { - return position_; -} diff --git a/lib/lexer/tokens/PunctToken.hpp b/lib/lexer/tokens/PunctToken.hpp deleted file mode 100644 index d7feafe..0000000 --- a/lib/lexer/tokens/PunctToken.hpp +++ /dev/null @@ -1,32 +0,0 @@ -#ifndef PUNCTTOKEN_HPP_ -#define PUNCTTOKEN_HPP_ - -#include - -#include "Token.hpp" -#include "TokenVisitor.hpp" - -class PunctToken final : public Token { -public: - PunctToken(char ch, const TokenPosition& position); - - PunctToken(std::string punct, const TokenPosition& position); - - [[nodiscard]] std::string GetStringType() const noexcept override; - - [[nodiscard]] std::string GetLexeme() const noexcept override; - - [[nodiscard]] TokenPtr Clone() const override; - - void Accept(TokenVisitor& visitor) const override; - - [[nodiscard]] std::string ToString() const override; - - [[nodiscard]] const TokenPosition& GetPosition() const noexcept override; - -private: - std::string lexeme_; - TokenPosition position_; -}; - -#endif // PUNCTTOKEN_HPP_ diff --git a/lib/lexer/tokens/Token.hpp b/lib/lexer/tokens/Token.hpp deleted file mode 100644 index 499b313..0000000 --- a/lib/lexer/tokens/Token.hpp +++ /dev/null @@ -1,29 +0,0 @@ -#ifndef TOKEN_HPP_ -#define TOKEN_HPP_ - -#include -#include - -#include "TokenPosition.hpp" -#include "TokenVisitor.hpp" - -class Token { // NOLINT(cppcoreguidelines-special-member-functions) -public: - virtual ~Token() = default; - - [[nodiscard]] virtual std::string GetStringType() const = 0; - - [[nodiscard]] virtual std::string GetLexeme() const = 0; - - [[nodiscard]] virtual std::shared_ptr Clone() const = 0; - - virtual void Accept(TokenVisitor& v) const = 0; - - [[nodiscard]] virtual std::string ToString() const = 0; - - [[nodiscard]] virtual const TokenPosition& GetPosition() const noexcept = 0; -}; - -using TokenPtr = std::shared_ptr; - -#endif // TOKEN_HPP_ diff --git a/lib/lexer/tokens/TokenFactory.cpp b/lib/lexer/tokens/TokenFactory.cpp deleted file mode 100644 index aca2384..0000000 --- a/lib/lexer/tokens/TokenFactory.cpp +++ /dev/null @@ -1,72 +0,0 @@ -#include "TokenFactory.hpp" - -#include "CommentToken.hpp" -#include "EofToken.hpp" -#include "IdentToken.hpp" -#include "KeywordToken.hpp" -#include "LiteralToken.hpp" -#include "NewlineToken.hpp" -#include "OperatorToken.hpp" -#include "PunctToken.hpp" -#include "lib/lexer/values/BoolValue.hpp" -#include "lib/lexer/values/CharValue.hpp" -#include "lib/lexer/values/FloatValue.hpp" -#include "lib/lexer/values/IntValue.hpp" -#include "lib/lexer/values/StringValue.hpp" - -namespace TokenFactory { - -TokenPtr MakeIdent(std::string lex, int32_t line, int32_t col) { - return std::make_shared(std::move(lex), TokenPosition(line, col)); -} - -TokenPtr MakeKeyword(std::string lex, int32_t line, int32_t col) { - return std::make_shared(std::move(lex), TokenPosition(line, col)); -} - -TokenPtr MakeOperator(std::string lex, int32_t line, int32_t col) { - return std::make_shared(std::move(lex), TokenPosition(line, col)); -} - -TokenPtr MakePunct(char ch, int32_t line, int32_t col) { - return std::make_shared(ch, TokenPosition(line, col)); -} - -TokenPtr MakePunct(std::string lex, int32_t line, int32_t col) { - return std::make_shared(std::move(lex), TokenPosition(line, col)); -} - -TokenPtr MakeNewline(int32_t line, int32_t col) { - return std::make_shared(TokenPosition(line, col)); -} - -TokenPtr MakeComment(std::string text, int32_t line, int32_t col) { - return std::make_shared(std::move(text), TokenPosition(line, col)); -} - -TokenPtr MakeEof(int32_t line, int32_t col) { - return std::make_shared(TokenPosition(line, col)); -} - -TokenPtr MakeIntLiteral(std::string raw, int64_t v, int32_t line, int32_t col) { - return std::make_shared(std::move(raw), std::make_unique(v), TokenPosition(line, col)); -} - -TokenPtr MakeFloatLiteral(std::string raw, double v, int32_t line, int32_t col) { - return std::make_shared(std::move(raw), std::make_unique(v), TokenPosition(line, col)); -} - -TokenPtr MakeStringLiteral(std::string raw, std::string s, int32_t line, int32_t col) { - return std::make_shared( - std::move(raw), std::make_unique(std::move(s)), TokenPosition(line, col)); -} - -TokenPtr MakeCharLiteral(std::string raw, char c, int32_t line, int32_t col) { - return std::make_shared(std::move(raw), std::make_unique(c), TokenPosition(line, col)); -} - -TokenPtr MakeBoolLiteral(std::string raw, bool b, int32_t line, int32_t col) { - return std::make_shared(std::move(raw), std::make_unique(b), TokenPosition(line, col)); -} - -}; // namespace TokenFactory diff --git a/lib/lexer/tokens/TokenFactory.hpp b/lib/lexer/tokens/TokenFactory.hpp deleted file mode 100644 index 24f460c..0000000 --- a/lib/lexer/tokens/TokenFactory.hpp +++ /dev/null @@ -1,38 +0,0 @@ -#ifndef TOKENFACTORY_HPP_ -#define TOKENFACTORY_HPP_ - -#include - -#include "Token.hpp" - -namespace TokenFactory { - -TokenPtr MakeIdent(std::string lex, int32_t line, int32_t col); - -TokenPtr MakeKeyword(std::string lex, int32_t line, int32_t col); - -TokenPtr MakeOperator(std::string lex, int32_t line, int32_t col); - -TokenPtr MakePunct(char ch, int32_t line, int32_t col); - -TokenPtr MakePunct(std::string lex, int32_t line, int32_t col); - -TokenPtr MakeNewline(int32_t line, int32_t col); - -TokenPtr MakeComment(std::string text, int32_t line, int32_t col); - -TokenPtr MakeEof(int32_t line, int32_t col); - -TokenPtr MakeIntLiteral(std::string raw, int64_t v, int32_t line, int32_t col); - -TokenPtr MakeFloatLiteral(std::string raw, double v, int32_t line, int32_t col); - -TokenPtr MakeStringLiteral(std::string raw, std::string s, int32_t line, int32_t col); - -TokenPtr MakeCharLiteral(std::string raw, char c, int32_t line, int32_t col); - -TokenPtr MakeBoolLiteral(std::string raw, bool b, int32_t line, int32_t col); - -}; // namespace TokenFactory - -#endif // TOKENFACTORY_HPP_ diff --git a/lib/lexer/tokens/TokenPosition.cpp b/lib/lexer/tokens/TokenPosition.cpp deleted file mode 100644 index 2c9dcec..0000000 --- a/lib/lexer/tokens/TokenPosition.cpp +++ /dev/null @@ -1,12 +0,0 @@ -#include "TokenPosition.hpp" - -TokenPosition::TokenPosition(int32_t line, int32_t column) : line_(line), column_(column) { -} - -int32_t TokenPosition::GetLine() const noexcept { - return line_; -} - -int32_t TokenPosition::GetColumn() const noexcept { - return column_; -} diff --git a/lib/lexer/tokens/TokenPosition.hpp b/lib/lexer/tokens/TokenPosition.hpp deleted file mode 100644 index 2b2be92..0000000 --- a/lib/lexer/tokens/TokenPosition.hpp +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef TOKENPOSITION_HPP_ -#define TOKENPOSITION_HPP_ - -#include - -class TokenPosition { -public: - TokenPosition(int32_t line, int32_t column); - - [[nodiscard]] int32_t GetLine() const noexcept; - - [[nodiscard]] int32_t GetColumn() const noexcept; - -private: - int32_t line_; - int32_t column_; -}; - -#endif // TOKENPOSITION_HPP_ diff --git a/lib/lexer/tokens/TokenVisitor.hpp b/lib/lexer/tokens/TokenVisitor.hpp deleted file mode 100644 index 323801b..0000000 --- a/lib/lexer/tokens/TokenVisitor.hpp +++ /dev/null @@ -1,34 +0,0 @@ -#ifndef TOKENVISITOR_HPP_ -#define TOKENVISITOR_HPP_ - -class EofToken; -class CommentToken; -class NewlineToken; -class PunctToken; -class OperatorToken; -class LiteralToken; -class KeywordToken; -class IdentToken; - -class TokenVisitor { // NOLINT(cppcoreguidelines-special-member-functions) -public: - virtual ~TokenVisitor() = default; - - virtual void Visit(const IdentToken& t) = 0; - - virtual void Visit(const KeywordToken& t) = 0; - - virtual void Visit(const LiteralToken& t) = 0; - - virtual void Visit(const OperatorToken& t) = 0; - - virtual void Visit(const PunctToken& t) = 0; - - virtual void Visit(const NewlineToken& t) = 0; - - virtual void Visit(const CommentToken& t) = 0; - - virtual void Visit(const EofToken& t) = 0; -}; - -#endif // TOKENVISITOR_HPP_ diff --git a/lib/lexer/values/BoolValue.cpp b/lib/lexer/values/BoolValue.cpp deleted file mode 100644 index 92fdf05..0000000 --- a/lib/lexer/values/BoolValue.cpp +++ /dev/null @@ -1,16 +0,0 @@ -#include "BoolValue.hpp" - -BoolValue::BoolValue(bool b) : v(b) { -} - -std::unique_ptr BoolValue::Clone() const { - return std::make_unique(v); -} - -std::string BoolValue::ToString() const { - return v ? "true" : "false"; -} - -std::string BoolValue::GetTypeName() const { - return "Bool"; -} diff --git a/lib/lexer/values/BoolValue.hpp b/lib/lexer/values/BoolValue.hpp deleted file mode 100644 index f51f3a9..0000000 --- a/lib/lexer/values/BoolValue.hpp +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef BOOLVALUE_HPP_ -#define BOOLVALUE_HPP_ - -#include -#include - -#include "Value.hpp" - -class BoolValue : public Value { -public: - explicit BoolValue(bool b); - - [[nodiscard]] std::unique_ptr Clone() const override; - - [[nodiscard]] std::string ToString() const override; - - [[nodiscard]] std::string GetTypeName() const override; - -private: - bool v; -}; - -#endif // BOOLVALUE_HPP_ diff --git a/lib/lexer/values/CharValue.cpp b/lib/lexer/values/CharValue.cpp deleted file mode 100644 index 4c7f584..0000000 --- a/lib/lexer/values/CharValue.cpp +++ /dev/null @@ -1,16 +0,0 @@ -#include "CharValue.hpp" - -CharValue::CharValue(char c) : v(c) { -} - -std::unique_ptr CharValue::Clone() const { - return std::make_unique(v); -} - -std::string CharValue::ToString() const { - return "'" + std::string(1, v) + "'"; -} - -std::string CharValue::GetTypeName() const { - return "Char"; -} diff --git a/lib/lexer/values/CharValue.hpp b/lib/lexer/values/CharValue.hpp deleted file mode 100644 index e111327..0000000 --- a/lib/lexer/values/CharValue.hpp +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef CHARVALUE_HPP_ -#define CHARVALUE_HPP_ - -#include -#include - -#include "Value.hpp" - -class CharValue : public Value { -public: - explicit CharValue(char c); - - [[nodiscard]] std::unique_ptr Clone() const override; - - [[nodiscard]] std::string ToString() const override; - - [[nodiscard]] std::string GetTypeName() const override; - -private: - char v; -}; - -#endif // CHARVALUE_HPP_ diff --git a/lib/lexer/values/FloatValue.cpp b/lib/lexer/values/FloatValue.cpp deleted file mode 100644 index a9c790f..0000000 --- a/lib/lexer/values/FloatValue.cpp +++ /dev/null @@ -1,20 +0,0 @@ -#include "FloatValue.hpp" - -#include - -FloatValue::FloatValue(double x) : v(x) { -} - -std::unique_ptr FloatValue::Clone() const { - return std::make_unique(v); -} - -std::string FloatValue::ToString() const { - std::ostringstream os; - os << v; - return os.str(); -} - -std::string FloatValue::GetTypeName() const { - return "Float"; -} diff --git a/lib/lexer/values/FloatValue.hpp b/lib/lexer/values/FloatValue.hpp deleted file mode 100644 index 77c8e1c..0000000 --- a/lib/lexer/values/FloatValue.hpp +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef FLOATVALUE_HPP_ -#define FLOATVALUE_HPP_ - -#include -#include - -#include "Value.hpp" - -class FloatValue : public Value { -public: - explicit FloatValue(double x); - - [[nodiscard]] std::unique_ptr Clone() const override; - - [[nodiscard]] std::string ToString() const override; - - [[nodiscard]] std::string GetTypeName() const override; - -private: - double v; -}; - -#endif // FLOATVALUE_HPP_ diff --git a/lib/lexer/values/IntValue.cpp b/lib/lexer/values/IntValue.cpp deleted file mode 100644 index d4e21df..0000000 --- a/lib/lexer/values/IntValue.cpp +++ /dev/null @@ -1,16 +0,0 @@ -#include "IntValue.hpp" - -IntValue::IntValue(int64_t x) : v(x) { -} - -std::unique_ptr IntValue::Clone() const { - return std::make_unique(v); -} - -std::string IntValue::ToString() const { - return std::to_string(v); -} - -std::string IntValue::GetTypeName() const { - return "Int"; -} diff --git a/lib/lexer/values/IntValue.hpp b/lib/lexer/values/IntValue.hpp deleted file mode 100644 index 45a3194..0000000 --- a/lib/lexer/values/IntValue.hpp +++ /dev/null @@ -1,24 +0,0 @@ -#ifndef INTVALUE_HPP_ -#define INTVALUE_HPP_ - -#include -#include -#include - -#include "Value.hpp" - -class IntValue : public Value { -public: - explicit IntValue(int64_t x); - - [[nodiscard]] std::unique_ptr Clone() const override; - - [[nodiscard]] std::string ToString() const override; - - [[nodiscard]] std::string GetTypeName() const override; - -private: - int64_t v; -}; - -#endif // INTVALUE_HPP_ diff --git a/lib/lexer/values/StringValue.cpp b/lib/lexer/values/StringValue.cpp deleted file mode 100644 index a5dbbbf..0000000 --- a/lib/lexer/values/StringValue.cpp +++ /dev/null @@ -1,18 +0,0 @@ -#include "StringValue.hpp" - -#include - -StringValue::StringValue(std::string s) : v(std::move(s)) { -} - -std::unique_ptr StringValue::Clone() const { - return std::make_unique(v); -} - -std::string StringValue::ToString() const { - return "\"" + v + "\""; -} - -std::string StringValue::GetTypeName() const { - return "String"; -} diff --git a/lib/lexer/values/StringValue.hpp b/lib/lexer/values/StringValue.hpp deleted file mode 100644 index c4d04ce..0000000 --- a/lib/lexer/values/StringValue.hpp +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef STRINGVALUE_HPP_ -#define STRINGVALUE_HPP_ - -#include -#include - -#include "Value.hpp" - -class StringValue : public Value { -public: - explicit StringValue(std::string s); - - [[nodiscard]] std::unique_ptr Clone() const override; - - [[nodiscard]] std::string ToString() const override; - - [[nodiscard]] std::string GetTypeName() const override; - -private: - std::string v; -}; - -#endif // STRINGVALUE_HPP_ diff --git a/lib/lexer/values/Value.hpp b/lib/lexer/values/Value.hpp deleted file mode 100644 index b6ad569..0000000 --- a/lib/lexer/values/Value.hpp +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef VALUE_HPP_ -#define VALUE_HPP_ - -#include -#include - -class Value { // NOLINT(cppcoreguidelines-special-member-functions) -public: - virtual ~Value() = default; - - [[nodiscard]] virtual std::unique_ptr Clone() const = 0; - - [[nodiscard]] virtual std::string ToString() const = 0; - - [[nodiscard]] virtual std::string GetTypeName() const = 0; -}; - -#endif // VALUE_HPP_ diff --git a/lib/preprocessor/CMakeLists.txt b/lib/preprocessor/CMakeLists.txt index 3631006..d0c84ec 100644 --- a/lib/preprocessor/CMakeLists.txt +++ b/lib/preprocessor/CMakeLists.txt @@ -26,6 +26,7 @@ target_include_directories(preprocessor target_link_libraries(preprocessor PUBLIC lexer + tokens ) target_include_directories(preprocessor PUBLIC ${PROJECT_SOURCE_DIR}) diff --git a/lib/preprocessor/Preprocessor.cpp b/lib/preprocessor/Preprocessor.cpp index 1d58687..6ff0857 100644 --- a/lib/preprocessor/Preprocessor.cpp +++ b/lib/preprocessor/Preprocessor.cpp @@ -1,6 +1,5 @@ #include "Preprocessor.hpp" -#include #include #include @@ -32,15 +31,15 @@ std::expected, PreprocessorError> Preprocessor::Process() return std::unexpected(FileReadError(file.string(), "Read error")); } - Lexer lexer(content, false); - std::vector tokens; + lexer::Lexer lexer(content, false); + auto tokens_result = lexer.Tokenize(); - try { - tokens = std::move(lexer.Tokenize()); - } catch (const std::exception& e) { - return std::unexpected(PreprocessorError(e.what())); + if (!tokens_result) { + return std::unexpected(PreprocessorError(tokens_result.error().what())); } + std::vector tokens = std::move(tokens_result.value()); + for (std::unique_ptr& processor : token_processors_) { std::expected, PreprocessorError> processor_result = std::move(processor->Process(tokens)); diff --git a/lib/preprocessor/TokenProcessor.hpp b/lib/preprocessor/TokenProcessor.hpp index 2224f23..d52a9f5 100644 --- a/lib/preprocessor/TokenProcessor.hpp +++ b/lib/preprocessor/TokenProcessor.hpp @@ -4,11 +4,14 @@ #include #include +#include + #include "PreprocessorError.hpp" -#include "lib/lexer/tokens/Token.hpp" namespace ovum::compiler::preprocessor { +using TokenPtr = ovum::TokenPtr; + class TokenProcessor { // NOLINT(cppcoreguidelines-special-member-functions) public: virtual ~TokenProcessor() = default; diff --git a/lib/preprocessor/directives_processor/TokenDirectivesProcessor.hpp b/lib/preprocessor/directives_processor/TokenDirectivesProcessor.hpp index 25f3e23..acdd820 100644 --- a/lib/preprocessor/directives_processor/TokenDirectivesProcessor.hpp +++ b/lib/preprocessor/directives_processor/TokenDirectivesProcessor.hpp @@ -7,7 +7,8 @@ #include #include -#include "lib/lexer/tokens/Token.hpp" +#include + #include "lib/preprocessor/PreprocessorError.hpp" #include "lib/preprocessor/TokenProcessor.hpp" #include "lib/preprocessor/directives_processor/handlers/DirectiveHandler.hpp" diff --git a/lib/preprocessor/directives_processor/handlers/DefaultHandler.hpp b/lib/preprocessor/directives_processor/handlers/DefaultHandler.hpp index a3dc344..edf94cc 100644 --- a/lib/preprocessor/directives_processor/handlers/DefaultHandler.hpp +++ b/lib/preprocessor/directives_processor/handlers/DefaultHandler.hpp @@ -1,8 +1,9 @@ #ifndef PREPROCESSOR_DEFAULT_HANDLER_HPP_ #define PREPROCESSOR_DEFAULT_HANDLER_HPP_ +#include + #include "DirectiveHandler.hpp" -#include "lib/lexer/tokens/Token.hpp" namespace ovum::compiler::preprocessor { diff --git a/lib/preprocessor/directives_processor/handlers/DefineHandler.hpp b/lib/preprocessor/directives_processor/handlers/DefineHandler.hpp index 3ac0957..031fb36 100644 --- a/lib/preprocessor/directives_processor/handlers/DefineHandler.hpp +++ b/lib/preprocessor/directives_processor/handlers/DefineHandler.hpp @@ -1,8 +1,9 @@ #ifndef PREPROCESSOR_DEFINE_HANDLER_HPP_ #define PREPROCESSOR_DEFINE_HANDLER_HPP_ +#include + #include "DirectiveHandler.hpp" -#include "lib/lexer/tokens/Token.hpp" namespace ovum::compiler::preprocessor { diff --git a/lib/preprocessor/directives_processor/handlers/DirectiveHandler.hpp b/lib/preprocessor/directives_processor/handlers/DirectiveHandler.hpp index e182e86..af1998c 100644 --- a/lib/preprocessor/directives_processor/handlers/DirectiveHandler.hpp +++ b/lib/preprocessor/directives_processor/handlers/DirectiveHandler.hpp @@ -7,7 +7,8 @@ #include #include -#include "lib/lexer/tokens/Token.hpp" +#include + #include "lib/preprocessor/PreprocessorError.hpp" namespace ovum::compiler::preprocessor { diff --git a/lib/preprocessor/directives_processor/handlers/ElseHandler.hpp b/lib/preprocessor/directives_processor/handlers/ElseHandler.hpp index c4d3af9..597d50a 100644 --- a/lib/preprocessor/directives_processor/handlers/ElseHandler.hpp +++ b/lib/preprocessor/directives_processor/handlers/ElseHandler.hpp @@ -1,8 +1,9 @@ #ifndef PREPROCESSOR_ELSE_HANDLER_HPP_ #define PREPROCESSOR_ELSE_HANDLER_HPP_ +#include + #include "DirectiveHandler.hpp" -#include "lib/lexer/tokens/Token.hpp" namespace ovum::compiler::preprocessor { diff --git a/lib/preprocessor/directives_processor/handlers/EndifHandler.hpp b/lib/preprocessor/directives_processor/handlers/EndifHandler.hpp index 88d1e5f..eeb5ae4 100644 --- a/lib/preprocessor/directives_processor/handlers/EndifHandler.hpp +++ b/lib/preprocessor/directives_processor/handlers/EndifHandler.hpp @@ -1,8 +1,9 @@ #ifndef PREPROCESSOR_ENDIF_HANDLER_HPP_ #define PREPROCESSOR_ENDIF_HANDLER_HPP_ +#include + #include "DirectiveHandler.hpp" -#include "lib/lexer/tokens/Token.hpp" namespace ovum::compiler::preprocessor { diff --git a/lib/preprocessor/directives_processor/handlers/IfdefHandler.hpp b/lib/preprocessor/directives_processor/handlers/IfdefHandler.hpp index eeebe2f..86271af 100644 --- a/lib/preprocessor/directives_processor/handlers/IfdefHandler.hpp +++ b/lib/preprocessor/directives_processor/handlers/IfdefHandler.hpp @@ -1,8 +1,9 @@ #ifndef PREPROCESSOR_IFDEF_HANDLER_HPP_ #define PREPROCESSOR_IFDEF_HANDLER_HPP_ +#include + #include "DirectiveHandler.hpp" -#include "lib/lexer/tokens/Token.hpp" namespace ovum::compiler::preprocessor { diff --git a/lib/preprocessor/directives_processor/handlers/IfndefHandler.hpp b/lib/preprocessor/directives_processor/handlers/IfndefHandler.hpp index 81468f9..431be7f 100644 --- a/lib/preprocessor/directives_processor/handlers/IfndefHandler.hpp +++ b/lib/preprocessor/directives_processor/handlers/IfndefHandler.hpp @@ -1,8 +1,9 @@ #ifndef PREPROCESSOR_IFNDEF_HANDLER_HPP_ #define PREPROCESSOR_IFNDEF_HANDLER_HPP_ +#include + #include "DirectiveHandler.hpp" -#include "lib/lexer/tokens/Token.hpp" namespace ovum::compiler::preprocessor { diff --git a/lib/preprocessor/directives_processor/handlers/UndefHandler.hpp b/lib/preprocessor/directives_processor/handlers/UndefHandler.hpp index 57a7d22..5b7616b 100644 --- a/lib/preprocessor/directives_processor/handlers/UndefHandler.hpp +++ b/lib/preprocessor/directives_processor/handlers/UndefHandler.hpp @@ -1,8 +1,9 @@ #ifndef PREPROCESSOR_UNDEF_HANDLER_HPP_ #define PREPROCESSOR_UNDEF_HANDLER_HPP_ +#include + #include "DirectiveHandler.hpp" -#include "lib/lexer/tokens/Token.hpp" namespace ovum::compiler::preprocessor { diff --git a/lib/preprocessor/import_processor/TokenImportProcessor.cpp b/lib/preprocessor/import_processor/TokenImportProcessor.cpp index 4ae9f18..660b9b5 100644 --- a/lib/preprocessor/import_processor/TokenImportProcessor.cpp +++ b/lib/preprocessor/import_processor/TokenImportProcessor.cpp @@ -124,15 +124,16 @@ std::expected TokenImportProcessor::GatherDependencies( std::string content_str = std::move(content_result.value()); std::string_view content_view(content_str); - Lexer lexer(content_view, false); - std::vector raw_tokens; + lexer::Lexer lexer(content_view, false); + auto raw_tokens_result = lexer.Tokenize(); - try { - raw_tokens = lexer.Tokenize(); - } catch (const std::exception& e) { - return std::unexpected(PreprocessorError("Lexer error for " + dep_path.string() + ": " + e.what())); + if (!raw_tokens_result) { + return std::unexpected( + PreprocessorError("Lexer error for " + dep_path.string() + ": " + raw_tokens_result.error().what())); } + std::vector raw_tokens = std::move(raw_tokens_result.value()); + std::expected sub_result = GatherDependencies(dep_path, raw_tokens); if (!sub_result) { diff --git a/lib/preprocessor/import_processor/TokenImportProcessor.hpp b/lib/preprocessor/import_processor/TokenImportProcessor.hpp index ad75bfc..7f6f0f7 100644 --- a/lib/preprocessor/import_processor/TokenImportProcessor.hpp +++ b/lib/preprocessor/import_processor/TokenImportProcessor.hpp @@ -9,8 +9,9 @@ #include #include +#include + #include "FileGraph.hpp" -#include "lib/lexer/tokens/Token.hpp" #include "lib/preprocessor/PreprocessorError.hpp" #include "lib/preprocessor/TokenProcessor.hpp" diff --git a/tests/lexer_big_programs_tests.cpp b/tests/lexer_big_programs_tests.cpp index 574bd12..a291fa1 100644 --- a/tests/lexer_big_programs_tests.cpp +++ b/tests/lexer_big_programs_tests.cpp @@ -4,6 +4,8 @@ #include "lib/lexer/Lexer.hpp" #include "test_suites/LexerUnitTestSuite.hpp" +using ovum::compiler::lexer::Lexer; + TEST(LexerUnitTestSuite, ExampleFundamentals) { const std::string src = R"OVUM(fun ExampleFundamentals(): Void { val i: int = 42 @@ -15,8 +17,9 @@ TEST(LexerUnitTestSuite, ExampleFundamentals) { sys::Print(i.ToString()) })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun","ExampleFundamentals","(",")",":","Void","{","\\n", @@ -53,8 +56,9 @@ val count: Int = 0 val pi: Float = 3.14 })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun","ExampleReferences","(",")",":","Void","{","\\n", @@ -90,8 +94,9 @@ val optArr: IntArray? = null val safeInt: Int? = 42 })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "ExampleNullable", "(", ")", ":", "Void", "{", "\\n", @@ -118,8 +123,9 @@ intArr[1] := 2 val emptyList: IntArray = IntArray(0) })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun","ExampleArrays","(",")",":","Void","{","\\n", @@ -153,8 +159,9 @@ val shape: IShape = Circle(5.0) val circle: Circle? = Circle(10.0) })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "interface", "IShape", "{", "fun", "Area", "(", ")", ":", @@ -195,8 +202,9 @@ val obj: ExampleClass = ExampleClass() obj.mutable = 30.0 })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "class","ExampleClass","{","\\n", @@ -225,8 +233,9 @@ else if (x < 0) { sys::Print("Negative") } else { sys::Print("Zero") } })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun","SimpleIf","(","x",":","int",")",":","Void","{","\\n", @@ -254,8 +263,9 @@ else if (!a ^ b) { sys::Print("Exactly one false") } else { sys::Print("Both false") } })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun","BooleanConditions","(","a",":","bool",",","b",":","bool",")",":","Void","{","\\n", @@ -287,8 +297,9 @@ else if (obj is Int) { val num: Int = obj as Int sys::Print(num.ToString()) } else { sys::Print("Unknown type") } })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "TypeConditions", "(", "obj", ":", "Object", ")", ":", "Void", "{", @@ -326,8 +337,9 @@ if (value > 10) { return value * 2 } else { return value } })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "ElvisInIf", "(", "opt", ":", "Int", "?", ")", ":", "int", "{", "\\n", "val", "value", @@ -350,8 +362,9 @@ else if (x > 10) return "Big" else return "Other" })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "WhenLike", "(", "x", ":", "int", ")", ":", "String", "{", "\\n", "if", @@ -381,8 +394,9 @@ while (counter < n) { return counter })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "WhileExample", "(", "n", ":", "int", ")", ":", "int", "{", @@ -423,8 +437,9 @@ for (num in arr) { if (num < 0) break sum = sum + num } return sum })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "ForExample", "(", "arr", ":", "IntArray", ")", ":", "int", "{", "\\n", @@ -453,8 +468,9 @@ if (i > 5) break } } })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "NestedLoops", "(", "matrix", ":", "IntArrayArray", ")", ":", "Void", "{", @@ -501,8 +517,9 @@ while (i < arr.Length()) { sys::Print((i.ToString() + ": " + arr[i].ToString())) i = i + 1 }})OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "ForWithIndex", "(", "arr", ":", "IntArray", ")", ":", "Void", "{", @@ -541,8 +558,9 @@ fromLiteral[0] := 1 fromLiteral[1] := 2 })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "ListCreation", "(", ")", ":", "Void", "{", @@ -590,8 +608,9 @@ if (list[idx] % 2 == 0) { list[idx] := list[idx] * 2 } idx = idx + 1 } return total })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "ListIteration", "(", "list", ":", "IntArray", ")", ":", "int", "{", @@ -633,8 +652,9 @@ newList[original.Length()] := 100 return newList })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "ListOps", "(", "original", ":", "IntArray", ")", ":", "IntArray", "{", "\\n", @@ -665,8 +685,9 @@ for (num in list) { if (num == target) { return Int(num) } } return null })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "FindInList", "(", "list", ":", "IntArray", ",", "target", ":", "int", ")", ":", "Int", "?", "{", @@ -694,8 +715,9 @@ val length: int = obj?.Length() ?: 0 sys::Print(length.ToString()) })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "SafeCall", "(", "obj", ":", "String", "?", ")", ":", "Void", "{", "\\n", "val", @@ -717,8 +739,9 @@ val str: String = optInt?.ToString() ?: "Unknown" return default })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "ElvisExamples", "(", "optInt", ":", "Int", "?", ")", ":", "int", "{", @@ -748,8 +771,9 @@ val safe: Object? = opt if (safe == null) { sys::Print("Is null") } })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "NullChecks", "(", "opt", ":", "Object", "?", ")", ":", @@ -781,8 +805,9 @@ sys::Print(value.ToString()) } val risky: Int = obj as Int })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "CastNullable", "(", "obj", ":", "Object", "?", ")", ":", "Void", "{", @@ -820,8 +845,9 @@ while (i < src.Length()) { copy[i] := src[i] i = i + 1 } return copy })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "CopyNullable", "(", "src", ":", "IntArray", "?", ")", ":", "IntArray", "?", "{", @@ -862,8 +888,9 @@ val len: int = nested?.Inner?.Length() ?: 0 sys::Print(len.ToString()) })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "class", "Nested", "{", @@ -901,8 +928,9 @@ val bVal: int = b ?: 0 return Int(aVal + bVal) })OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "pure", "fun", "SafeAdd", "(", "a", ":", "Int", "?", ",", "b", ":", "Int", "?", ")", ":", @@ -925,8 +953,9 @@ if (ptr != null) { val bytes: ByteArray = (ptr as ByteArray) } }})OVUM"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); // clang-format off std::vector expected_lexemes = { "fun", "UnsafeNullable", "(", "ptr", ":", "Pointer", "?", ")", ":", "Void", "{", diff --git a/tests/lexer_tests.cpp b/tests/lexer_tests.cpp index f6a9fb4..b158d2f 100644 --- a/tests/lexer_tests.cpp +++ b/tests/lexer_tests.cpp @@ -4,11 +4,14 @@ #include "lib/lexer/Lexer.hpp" #include "test_suites/LexerUnitTestSuite.hpp" +using ovum::compiler::lexer::Lexer; + TEST(LexerUnitTestSuite, EmptyString) { const std::string src = ""; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {}; std::vector expected_types = {}; LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -17,8 +20,9 @@ TEST(LexerUnitTestSuite, EmptyString) { TEST(LexerUnitTestSuite, SingleCharacter) { const std::string src = "c"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"c"}; std::vector expected_types = {"IDENT"}; LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -29,8 +33,9 @@ TEST(LexerUnitTestSuite, Keywords) { "fun pure val var class interface implements override if else while for in return break continue unsafe is as " "typealias"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = { "fun", "pure", "val", "var", "class", "interface", "implements", "override", "if", "else", "while", "for", "in", "return", "break", "continue", "unsafe", "is", "as", "typealias"}; @@ -41,8 +46,9 @@ TEST(LexerUnitTestSuite, Keywords) { TEST(LexerUnitTestSuite, Punctuation) { const std::string src = "{ } ( ) [ ] , : ;"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"{", "}", "(", ")", "[", "]", ",", ":", ";"}; std::vector expected_types(expected_lexemes.size(), "PUNCT"); LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -51,8 +57,9 @@ TEST(LexerUnitTestSuite, Punctuation) { TEST(LexerUnitTestSuite, Identifiers) { const std::string src = "abc ABC _abc ovum ExampleFundamentals Void String IntArray"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = { "abc", "ABC", "_abc", "ovum", "ExampleFundamentals", "Void", "String", "IntArray"}; std::vector expected_types(expected_lexemes.size(), "IDENT"); @@ -62,8 +69,9 @@ TEST(LexerUnitTestSuite, Identifiers) { TEST(LexerUnitTestSuite, OperatorsSimple) { const std::string src = "+ - * / % < > ! . ?"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"+", "-", "*", "/", "%", "<", ">", "!", ".", "?"}; std::vector expected_types(expected_lexemes.size(), "OPERATOR"); LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -72,8 +80,9 @@ TEST(LexerUnitTestSuite, OperatorsSimple) { TEST(LexerUnitTestSuite, OperatorsMultiChar) { const std::string src = "== != <= >= && || = := ::"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"==", "!=", "<=", ">=", "&&", "||", "=", ":=", "::"}; std::vector expected_types(expected_lexemes.size(), "OPERATOR"); LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -82,8 +91,9 @@ TEST(LexerUnitTestSuite, OperatorsMultiChar) { TEST(LexerUnitTestSuite, OperatorsNullSafe) { const std::string src = "?. ?:"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"?.", "?:"}; std::vector expected_types(expected_lexemes.size(), "OPERATOR"); LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -92,8 +102,9 @@ TEST(LexerUnitTestSuite, OperatorsNullSafe) { TEST(LexerUnitTestSuite, OperatorXor) { const std::string src = "^"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"^"}; std::vector expected_types = {"OPERATOR"}; LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -102,8 +113,9 @@ TEST(LexerUnitTestSuite, OperatorXor) { TEST(LexerUnitTestSuite, LiteralsInt) { const std::string src = "0 42 255"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"0", "42", "255"}; std::vector expected_types(expected_lexemes.size(), "LITERAL:Int"); LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -112,8 +124,9 @@ TEST(LexerUnitTestSuite, LiteralsInt) { TEST(LexerUnitTestSuite, LiteralsFloat) { const std::string src = "3.14 1e3 .5 5. 2.0E-2"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"3.14", "1e3", ".5", "5.", "2.0E-2"}; std::vector expected_types(expected_lexemes.size(), "LITERAL:Float"); LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -122,8 +135,9 @@ TEST(LexerUnitTestSuite, LiteralsFloat) { TEST(LexerUnitTestSuite, LiteralsChar) { const std::string src = "'A' '\\n'"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"'A'", "'\\n'"}; std::vector expected_types(expected_lexemes.size(), "LITERAL:Char"); LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -132,8 +146,9 @@ TEST(LexerUnitTestSuite, LiteralsChar) { TEST(LexerUnitTestSuite, LiteralsBool) { const std::string src = "true false"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"true", "false"}; std::vector expected_types(expected_lexemes.size(), "LITERAL:Bool"); LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -142,8 +157,9 @@ TEST(LexerUnitTestSuite, LiteralsBool) { TEST(LexerUnitTestSuite, KeywordNull) { const std::string src = "null"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"null"}; std::vector expected_types = {"KEYWORD"}; LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -152,8 +168,9 @@ TEST(LexerUnitTestSuite, KeywordNull) { TEST(LexerUnitTestSuite, LiteralsString) { const std::string src = R"("hello" "world\n" "")"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {R"("hello")", R"("world\n")", R"("")"}; std::vector expected_types(expected_lexemes.size(), "LITERAL:String"); LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -162,8 +179,9 @@ TEST(LexerUnitTestSuite, LiteralsString) { TEST(LexerUnitTestSuite, Preprocessor) { const std::string src = "#import #define #ifdef"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"#import", "#define", "#ifdef"}; std::vector expected_types(expected_lexemes.size(), "KEYWORD"); LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -172,8 +190,9 @@ TEST(LexerUnitTestSuite, Preprocessor) { TEST(LexerUnitTestSuite, Newline) { const std::string src = "\n"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"\\n"}; std::vector expected_types = {"NEWLINE"}; LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -182,8 +201,9 @@ TEST(LexerUnitTestSuite, Newline) { TEST(LexerUnitTestSuite, LiteralsIntHex) { const std::string src = "0x1A 0xFF"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"0x1A", "0xFF"}; std::vector expected_types(expected_lexemes.size(), "LITERAL:Int"); LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -192,8 +212,9 @@ TEST(LexerUnitTestSuite, LiteralsIntHex) { TEST(LexerUnitTestSuite, LiteralsIntBinary) { const std::string src = "0b1010 0b1111"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"0b1010", "0b1111"}; std::vector expected_types(expected_lexemes.size(), "LITERAL:Int"); LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -202,8 +223,9 @@ TEST(LexerUnitTestSuite, LiteralsIntBinary) { TEST(LexerUnitTestSuite, LiteralsIntNegative) { const std::string src = "-17 -42"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"-", "17", "-", "42"}; std::vector expected_types = {"OPERATOR", "LITERAL:Int", "OPERATOR", "LITERAL:Int"}; LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -212,8 +234,9 @@ TEST(LexerUnitTestSuite, LiteralsIntNegative) { TEST(LexerUnitTestSuite, LiteralsFloatSpecial) { const std::string src = "Infinity -Infinity NaN"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"Infinity", "-", "Infinity", "NaN"}; std::vector expected_types = {"LITERAL:Float", "OPERATOR", "LITERAL:Float", "LITERAL:Float"}; LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -222,8 +245,9 @@ TEST(LexerUnitTestSuite, LiteralsFloatSpecial) { TEST(LexerUnitTestSuite, LiteralsCharEscapes) { const std::string src = R"('A' '\n' '\t' '\0' '\'')"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"'A'", "'\\n'", "'\\t'", "'\\0'", "'\\''"}; std::vector expected_types(expected_lexemes.size(), "LITERAL:Char"); LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -232,8 +256,9 @@ TEST(LexerUnitTestSuite, LiteralsCharEscapes) { TEST(LexerUnitTestSuite, WhitespaceSkipping) { const std::string src = "fun val \t if"; Lexer lexer(src); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"fun", "val", "if"}; std::vector expected_types(expected_lexemes.size(), "KEYWORD"); LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -243,8 +268,9 @@ TEST(LexerUnitTestSuite, SingleLineComment) { const std::string src = R"OVUM(// This is a single-line comment fun Main)OVUM"; Lexer lexer(src, true); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {" This is a single-line comment", "\\n", "fun", "Main"}; std::vector expected_types = {"COMMENT", "NEWLINE", "KEYWORD", "IDENT"}; LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -254,8 +280,9 @@ TEST(LexerUnitTestSuite, SingleLineCommentNoSpace) { const std::string src = R"OVUM(//This is a single-line comment without space )OVUM"; Lexer lexer(src, true); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {"This is a single-line comment without space", "\\n"}; std::vector expected_types = {"COMMENT", "NEWLINE"}; LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -265,8 +292,9 @@ TEST(LexerUnitTestSuite, MultiLineCommentPositive) { const std::string src = R"OVUM(/* This is a properly closed multi-line comment */)OVUM"; Lexer lexer(src, true); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {" This is a properly closed\nmulti-line comment "}; std::vector expected_types = {"COMMENT"}; LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -276,8 +304,9 @@ TEST(LexerUnitTestSuite, MultiLineCommentWithSlashesAndStars) { const std::string src = R"OVUM(/* This comment contains / and * signs: /* // */ fun Main())OVUM"; Lexer lexer(src, true); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = { " This comment contains / and * signs: /* // ", "\\n", "fun", "Main", "(", ")"}; std::vector expected_types = {"COMMENT", "NEWLINE", "KEYWORD", "IDENT", "PUNCT", "PUNCT"}; @@ -287,8 +316,9 @@ fun Main())OVUM"; TEST(LexerUnitTestSuite, IncorrectCommentWithSlashesAndStars) { const std::string src = R"OVUM(/* This comment contains / and * signs: /* // */ */)OVUM"; Lexer lexer(src, true); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {" This comment contains / and * signs: /* // ", "*", "/"}; std::vector expected_types = {"COMMENT", "OPERATOR", "OPERATOR"}; LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -301,8 +331,9 @@ Third line after empty line Fifth line */)OVUM"; Lexer lexer(src, true); - auto tokens = lexer.Tokenize(); - auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens); + auto tokens_result = lexer.Tokenize(); + ASSERT_TRUE(tokens_result.has_value()) << "Tokenize failed: " << tokens_result.error().what(); + auto items = LexerUnitTestSuite::ExtractLexemesAndTypes(tokens_result.value()); std::vector expected_lexemes = {" First line\n\nThird line after empty line\n\nFifth line "}; std::vector expected_types = {"COMMENT"}; LexerUnitTestSuite::AssertLexemesAndTypesEqual(items, expected_lexemes, expected_types); @@ -313,91 +344,106 @@ Fifth line */)OVUM"; TEST(LexerUnitTestSuite, InvalidCharacter) { const std::string src = "fun @main(): Void {}"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error for invalid character"; } TEST(LexerUnitTestSuite, InvalidCharacterInIdentifier) { const std::string src = "val name$var: int = 0"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } TEST(LexerUnitTestSuite, InvalidIdentifierStartingWithDigit) { const std::string src = "val 1abc: int = 0"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } TEST(LexerUnitTestSuite, InvalidHexNumber) { const std::string src = "val x: int = 0xG"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } TEST(LexerUnitTestSuite, InvalidBinaryNumber) { const std::string src = "val x: int = 0b2"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } TEST(LexerUnitTestSuite, InvalidFloatNumber) { const std::string src = "val x: float = 1.2.3"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } TEST(LexerUnitTestSuite, InvalidFloatIncompleteExp) { const std::string src = "val x: float = 1e"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } TEST(LexerUnitTestSuite, UnterminatedString) { const std::string src = R"(val msg: String = "hello)"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } TEST(LexerUnitTestSuite, InvalidEscapeInString) { const std::string src = R"(val msg: String = "\z")"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } TEST(LexerUnitTestSuite, UnterminatedChar) { const std::string src = "val c: char = 'A"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } TEST(LexerUnitTestSuite, EmptyCharLiteral) { const std::string src = R"(val c: char = '')"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } TEST(LexerUnitTestSuite, MultiCharLiteral) { const std::string src = "val c: char = 'AB'"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } TEST(LexerUnitTestSuite, InvalidEscapeInChar) { const std::string src = "val c: char = '\\z'"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } TEST(LexerUnitTestSuite, InvalidPreprocessor) { const std::string src = "#invalid"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } TEST(LexerUnitTestSuite, MultipleErrorsInOneFile) { const std::string src = R"(fun main@(): Void { val x = "unclosed; val y = 0xG })"; Lexer lexer(src); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } TEST(LexerUnitTestSuite, MultiLineCommentNegativeUnclosed) { @@ -406,5 +452,6 @@ fun Main(): Void { val x: int = 42 })OVUM"; Lexer lexer(src, true); - ASSERT_THROW(lexer.Tokenize(), std::runtime_error); + auto result = lexer.Tokenize(); + ASSERT_FALSE(result.has_value()) << "Expected lexer error"; } diff --git a/tests/test_suites/LexerUnitTestSuite.cpp b/tests/test_suites/LexerUnitTestSuite.cpp index 86c4e31..9591cf4 100644 --- a/tests/test_suites/LexerUnitTestSuite.cpp +++ b/tests/test_suites/LexerUnitTestSuite.cpp @@ -9,7 +9,7 @@ void LexerUnitTestSuite::TearDown() { } std::vector> LexerUnitTestSuite::ExtractLexemesAndTypes( - const std::vector& tokens) { + const std::vector& tokens) { std::vector> out; if (tokens.empty()) return out; diff --git a/tests/test_suites/LexerUnitTestSuite.hpp b/tests/test_suites/LexerUnitTestSuite.hpp index eb5a9ff..217f296 100644 --- a/tests/test_suites/LexerUnitTestSuite.hpp +++ b/tests/test_suites/LexerUnitTestSuite.hpp @@ -2,10 +2,13 @@ #define OVUMC_LEXERUNITTESTSUITE_HPP #include -#include "lib/lexer/tokens/Token.hpp" + +#include struct LexerUnitTestSuite : public testing::Test { - static std::vector> ExtractLexemesAndTypes(const std::vector& tokens); +public: + static std::vector> ExtractLexemesAndTypes( + const std::vector& tokens); static void PrintLexemesAndTypes(const std::vector>& items); diff --git a/tests/test_suites/PreprocessorUnitTestSuite.cpp b/tests/test_suites/PreprocessorUnitTestSuite.cpp index 030ea9d..df105e6 100644 --- a/tests/test_suites/PreprocessorUnitTestSuite.cpp +++ b/tests/test_suites/PreprocessorUnitTestSuite.cpp @@ -76,10 +76,14 @@ std::expected, std::string> PreprocessorUnitTestSuite::Tok std::string content((std::istreambuf_iterator(file)), std::istreambuf_iterator()); - Lexer lexer(content, false); + ovum::compiler::lexer::Lexer lexer(content, false); auto tokens_result = lexer.Tokenize(); - return tokens_result; + if (!tokens_result) { + return std::unexpected("Lexer error: " + std::string(tokens_result.error().what())); + } + + return tokens_result.value(); } bool PreprocessorUnitTestSuite::CompareTokenSequences(const std::vector& actual, diff --git a/tests/test_suites/PreprocessorUnitTestSuite.hpp b/tests/test_suites/PreprocessorUnitTestSuite.hpp index 533df9a..0c1d7a6 100644 --- a/tests/test_suites/PreprocessorUnitTestSuite.hpp +++ b/tests/test_suites/PreprocessorUnitTestSuite.hpp @@ -6,7 +6,8 @@ #include #include -#include "lib/lexer/tokens/Token.hpp" +#include + #include "lib/preprocessor/PreprocessorError.hpp" namespace ovum::compiler::preprocessor { diff --git a/tests/test_suites/ProjectIntegrationTestSuite.hpp b/tests/test_suites/ProjectIntegrationTestSuite.hpp index 0ecab1e..1ac983e 100644 --- a/tests/test_suites/ProjectIntegrationTestSuite.hpp +++ b/tests/test_suites/ProjectIntegrationTestSuite.hpp @@ -6,6 +6,7 @@ #include struct ProjectIntegrationTestSuite : public testing::Test { // special test structure +public: const std::string kTemporaryDirectoryName = "./gtest_tmp"; void SetUp() override; // method that is called at the beginning of every test