Lexer refactoring

cmake/IncludeExternalLibraries.cmake

@@ -11,3 +11,12 @@ FetchContent_Declare(
 )
 
 FetchContent_MakeAvailable(googletest)
+
+# Ovum Common
+FetchContent_Declare(
+    ovumcommon
+    GIT_REPOSITORY https://github.com/Ovum-Programming-Language/OvumCommon.git
+    GIT_TAG main
+)
+
+FetchContent_MakeAvailable(ovumcommon)

lib/lexer/CMakeLists.txt

@@ -13,21 +13,6 @@ add_library(lexer STATIC
         handlers/SlashHandler.cpp
         handlers/StringHandler.cpp
         handlers/WhitespaceHandler.cpp
-        tokens/CommentToken.cpp
-        tokens/EofToken.cpp
-        tokens/IdentToken.cpp
-        tokens/KeywordToken.cpp
-        tokens/LiteralToken.cpp
-        tokens/NewlineToken.cpp
-        tokens/OperatorToken.cpp
-        tokens/PunctToken.cpp
-        values/BoolValue.cpp
-        values/CharValue.cpp
-        values/FloatValue.cpp
-        values/IntValue.cpp
-        values/StringValue.cpp
-        tokens/TokenFactory.cpp
-        tokens/TokenPosition.cpp
 )
 
 target_include_directories(lexer
@@ -39,3 +24,7 @@ target_include_directories(lexer
         PRIVATE
         ${CMAKE_CURRENT_SOURCE_DIR}
 )
+
+target_link_libraries(lexer PUBLIC
+        tokens
+)

lib/lexer/Lexer.cpp

@@ -3,6 +3,8 @@
 #include <string>
 #include <utility>
 
+#include <tokens/TokenFactory.hpp>
+
 #include "handlers/CharHandler.hpp"
 #include "handlers/ColonHandler.hpp"
 #include "handlers/DefaultHandler.hpp"
@@ -16,7 +18,7 @@
 #include "handlers/StringHandler.hpp"
 #include "handlers/WhitespaceHandler.hpp"
 
-#include "tokens/TokenFactory.hpp"
+namespace ovum::compiler::lexer {
 
 namespace {
 constexpr const char* kOperatorChars = "+-*%<>=!&|^~?";
@@ -27,7 +29,7 @@ Lexer::Lexer(std::string_view src, bool keep_comments) :
     wrapper_(src, keep_comments), handlers_(MakeDefaultHandlers()), default_handler_(MakeDefaultHandler()) {
 }
 
-std::vector<TokenPtr> Lexer::Tokenize() {
+std::expected<std::vector<TokenPtr>, LexerError> Lexer::Tokenize() {
   std::vector<TokenPtr> tokens;
   tokens.reserve(kDefaultTokenReserve);
 
@@ -41,7 +43,13 @@ std::vector<TokenPtr> Lexer::Tokenize() {
       current_handler = default_handler_.get();
     }
 
-    OptToken maybe_token = current_handler->Scan(wrapper_);
+    auto maybe_token_result = current_handler->Scan(wrapper_);
+
+    if (!maybe_token_result) {
+      return std::unexpected(maybe_token_result.error());
+    }
+
+    OptToken maybe_token = maybe_token_result.value();
 
     if (maybe_token && *maybe_token) {
       tokens.push_back(std::move(*maybe_token));
@@ -105,3 +113,5 @@ std::array<std::unique_ptr<Handler>, kDefaultTokenReserve> Lexer::MakeDefaultHan
 std::unique_ptr<Handler> Lexer::MakeDefaultHandler() {
   return std::make_unique<DefaultHandler>();
 }
+
+} // namespace ovum::compiler::lexer

lib/lexer/Lexer.hpp

@@ -1,22 +1,30 @@
-#ifndef LEXER_HPP_
-#define LEXER_HPP_
+#ifndef LEXER_LEXER_HPP_
+#define LEXER_LEXER_HPP_
 
 #include <array>
 #include <cstddef>
+#include <expected>
 #include <memory>
 #include <string_view>
 #include <vector>
 
+#include <tokens/Token.hpp>
+
+#include "LexerError.hpp"
 #include "SourceCodeWrapper.hpp"
 #include "handlers/Handler.hpp"
 
+namespace ovum::compiler::lexer {
+
+using TokenPtr = ovum::TokenPtr;
+
 constexpr std::size_t kDefaultTokenReserve = 256;
 
 class Lexer {
 public:
   explicit Lexer(std::string_view src, bool keep_comments = false);
 
-  std::vector<TokenPtr> Tokenize();
+  std::expected<std::vector<TokenPtr>, LexerError> Tokenize();
 
   void SetHandler(unsigned char ch, std::unique_ptr<Handler> handler);
 
@@ -32,4 +40,6 @@ class Lexer {
   std::unique_ptr<Handler> default_handler_;
 };
 
-#endif // LEXER_HPP_
+} // namespace ovum::compiler::lexer
+
+#endif // LEXER_LEXER_HPP_

lib/lexer/LexerError.hpp

@@ -1,11 +1,15 @@
-#ifndef LEXERERROR_HPP_
-#define LEXERERROR_HPP_
+#ifndef LEXER_LEXERERROR_HPP_
+#define LEXER_LEXERERROR_HPP_
 
 #include <stdexcept>
 
+namespace ovum::compiler::lexer {
+
 class LexerError : public std::runtime_error {
 public:
   using std::runtime_error::runtime_error;
 };
 
-#endif // LEXERERROR_HPP_
+} // namespace ovum::compiler::lexer
+
+#endif // LEXER_LEXERERROR_HPP_

lib/lexer/SourceCodeWrapper.cpp

@@ -2,6 +2,8 @@
 
 #include <algorithm>
 
+namespace ovum::compiler::lexer {
+
 const std::unordered_set<std::string_view> SourceCodeWrapper::kKeywords = {
     "fun",        "class",   "interface", "var",    "override", "pure",    "if",     "else",      "continue",
     "break",      "for",     "while",     "return", "unsafe",   "val",     "static", "public",    "private",
@@ -121,3 +123,5 @@ bool SourceCodeWrapper::IsKeyword(std::string_view s) {
 bool SourceCodeWrapper::IsMultiOp(std::string_view s) {
   return kMultiOps.contains(s);
 }
+
+} // namespace ovum::compiler::lexer

lib/lexer/SourceCodeWrapper.hpp

@@ -1,12 +1,14 @@
-#ifndef SOURCECODEWRAPPER_HPP_
-#define SOURCECODEWRAPPER_HPP_
+#ifndef LEXER_SOURCECODEWRAPPER_HPP_
+#define LEXER_SOURCECODEWRAPPER_HPP_
 
 #include <cstdint>
 #include <functional>
 #include <string>
 #include <string_view>
 #include <unordered_set>
 
+namespace ovum::compiler::lexer {
+
 class SourceCodeWrapper {
 public:
   SourceCodeWrapper(std::string_view src, bool keep_comments = false);
@@ -53,4 +55,6 @@ class SourceCodeWrapper {
   static const std::unordered_set<std::string_view> kMultiOps;
 };
 
-#endif // SOURCECODEWRAPPER_HPP_
+} // namespace ovum::compiler::lexer
+
+#endif // LEXER_SOURCECODEWRAPPER_HPP_

lib/lexer/handlers/CharHandler.cpp

@@ -1,9 +1,12 @@
 #include "CharHandler.hpp"
 
+#include <tokens/TokenFactory.hpp>
+
 #include "lib/lexer/LexerError.hpp"
-#include "tokens/TokenFactory.hpp"
 
-OptToken CharHandler::Scan(SourceCodeWrapper& wrapper) {
+namespace ovum::compiler::lexer {
+
+std::expected<OptToken, LexerError> CharHandler::Scan(SourceCodeWrapper& wrapper) {
   std::string raw;
   raw.push_back('\'');
   char val = '\0';
@@ -30,7 +33,7 @@ OptToken CharHandler::Scan(SourceCodeWrapper& wrapper) {
         val = '\0';
         break;
       default:
-        throw LexerError(std::string("Unknown escape in char literal: \\") + e);
+        return std::unexpected(LexerError(std::string("Unknown escape in char literal: \\") + e));
     }
   } else {
     char c = wrapper.Advance();
@@ -39,20 +42,22 @@ OptToken CharHandler::Scan(SourceCodeWrapper& wrapper) {
   }
 
   if (wrapper.IsAtEnd()) {
-    throw LexerError("Unterminated char literal");
+    return std::unexpected(LexerError("Unterminated char literal"));
   }
 
   if (wrapper.Peek() == '\n') {
-    throw LexerError("Newline in char literal");
+    return std::unexpected(LexerError("Newline in char literal"));
   }
 
   if (wrapper.Peek() == '\'') {
     wrapper.Advance();
     raw.push_back('\'');
   } else {
-    throw LexerError("Too many characters in char literal");
+    return std::unexpected(LexerError("Too many characters in char literal"));
   }
 
   return std::make_optional(
       TokenFactory::MakeCharLiteral(std::move(raw), val, wrapper.GetLine(), wrapper.GetTokenCol()));
 }
+
+} // namespace ovum::compiler::lexer

lib/lexer/handlers/CharHandler.hpp

@@ -1,11 +1,15 @@
-#ifndef CHARHANDLER_HPP_
-#define CHARHANDLER_HPP_
+#ifndef LEXER_CHARHANDLER_HPP_
+#define LEXER_CHARHANDLER_HPP_
 
 #include "Handler.hpp"
 
+namespace ovum::compiler::lexer {
+
 class CharHandler : public Handler {
 public:
-  OptToken Scan(SourceCodeWrapper& wrapper) override;
+  std::expected<OptToken, LexerError> Scan(SourceCodeWrapper& wrapper) override;
 };
 
-#endif // CHARHANDLER_HPP_
+} // namespace ovum::compiler::lexer
+
+#endif // LEXER_CHARHANDLER_HPP_

lib/lexer/handlers/ColonHandler.cpp

@@ -1,7 +1,10 @@
 #include "ColonHandler.hpp"
-#include "lib/lexer/tokens/TokenFactory.hpp"
 
-OptToken ColonHandler::Scan(SourceCodeWrapper& wrapper) {
+#include <tokens/TokenFactory.hpp>
+
+namespace ovum::compiler::lexer {
+
+std::expected<OptToken, LexerError> ColonHandler::Scan(SourceCodeWrapper& wrapper) {
   std::string op;
   op.push_back(wrapper.CurrentChar());
 
@@ -18,3 +21,5 @@ OptToken ColonHandler::Scan(SourceCodeWrapper& wrapper) {
 
   return std::make_optional(TokenFactory::MakePunct(':', wrapper.GetLine(), wrapper.GetTokenCol()));
 }
+
+} // namespace ovum::compiler::lexer

lib/lexer/handlers/ColonHandler.hpp

@@ -1,10 +1,14 @@
-#ifndef COLONHANDLER_HPP_
-#define COLONHANDLER_HPP_
+#ifndef LEXER_COLONHANDLER_HPP_
+#define LEXER_COLONHANDLER_HPP_
 #include "Handler.hpp"
 
+namespace ovum::compiler::lexer {
+
 class ColonHandler : public Handler {
 public:
-  OptToken Scan(SourceCodeWrapper& wrapper) override;
+  std::expected<OptToken, LexerError> Scan(SourceCodeWrapper& wrapper) override;
 };
 
-#endif // COLONHANDLER_HPP_
+} // namespace ovum::compiler::lexer
+
+#endif // LEXER_COLONHANDLER_HPP_

lib/lexer/handlers/DefaultHandler.cpp

@@ -2,7 +2,11 @@
 
 #include "lib/lexer/LexerError.hpp"
 
-OptToken DefaultHandler::Scan(SourceCodeWrapper& wrapper) {
+namespace ovum::compiler::lexer {
+
+std::expected<OptToken, LexerError> DefaultHandler::Scan(SourceCodeWrapper& wrapper) {
   char c = wrapper.CurrentChar();
-  throw LexerError(std::string("Unexpected character: ") + c);
+  return std::unexpected(LexerError(std::string("Unexpected character: ") + c));
 }
+
+} // namespace ovum::compiler::lexer

lib/lexer/handlers/DefaultHandler.hpp

@@ -1,11 +1,15 @@
-#ifndef DEFAULTHANDLER_HPP_
-#define DEFAULTHANDLER_HPP_
+#ifndef LEXER_DEFAULTHANDLER_HPP_
+#define LEXER_DEFAULTHANDLER_HPP_
 
 #include "Handler.hpp"
 
+namespace ovum::compiler::lexer {
+
 class DefaultHandler : public Handler {
 public:
-  OptToken Scan(SourceCodeWrapper& wrapper) override;
+  std::expected<OptToken, LexerError> Scan(SourceCodeWrapper& wrapper) override;
 };
 
-#endif // DEFAULTHANDLER_HPP_
+} // namespace ovum::compiler::lexer
+
+#endif // LEXER_DEFAULTHANDLER_HPP_

lib/lexer/handlers/DotCompositeHandler.cpp

@@ -1,22 +1,26 @@
 #include "DotCompositeHandler.hpp"
 
+namespace ovum::compiler::lexer {
+
 DotCompositeHandler::DotCompositeHandler() :
     num_(std::make_unique<NumberHandler>()), op_(std::make_unique<OperatorHandler>()),
     punct_(std::make_unique<PunctHandler>()) {
 }
 
-OptToken DotCompositeHandler::Scan(SourceCodeWrapper& w) {
+std::expected<OptToken, LexerError> DotCompositeHandler::Scan(SourceCodeWrapper& w) {
   if (std::isdigit(static_cast<unsigned char>(w.Peek()))) {
     return num_->Scan(w);
   }
 
-  if (auto t = op_->Scan(w)) {
-    return t;
+  auto op_result = op_->Scan(w);
+  if (!op_result) {
+    return op_result;
   }
-
-  if (punct_) {
-    return punct_->Scan(w);
+  if (op_result.value()) {
+    return op_result;
   }
 
-  return std::nullopt;
+  return punct_->Scan(w);
 }
+
+} // namespace ovum::compiler::lexer

lib/lexer/handlers/DotCompositeHandler.hpp

@@ -1,21 +1,25 @@
-#ifndef DOTCOMPOSITEHANDLER_HPP_
-#define DOTCOMPOSITEHANDLER_HPP_
+#ifndef LEXER_DOTCOMPOSITEHANDLER_HPP_
+#define LEXER_DOTCOMPOSITEHANDLER_HPP_
 
 #include "Handler.hpp"
 #include "NumberHandler.hpp"
 #include "OperatorHandler.hpp"
 #include "PunctHandler.hpp"
 
+namespace ovum::compiler::lexer {
+
 class DotCompositeHandler : public Handler {
 public:
   DotCompositeHandler();
 
-  OptToken Scan(SourceCodeWrapper& wrapper) override;
+  std::expected<OptToken, LexerError> Scan(SourceCodeWrapper& wrapper) override;
 
 private:
   std::unique_ptr<NumberHandler> num_;
   std::unique_ptr<OperatorHandler> op_;
   std::unique_ptr<PunctHandler> punct_;
 };
 
-#endif // DOTCOMPOSITEHANDLER_HPP_
+} // namespace ovum::compiler::lexer
+
+#endif // LEXER_DOTCOMPOSITEHANDLER_HPP_