Lexer! Again :)

bin/CMakeLists.txt

@@ -1,7 +1,7 @@
 add_executable(${PROJECT_NAME} main.cpp)
 
 target_link_libraries(${PROJECT_NAME} PUBLIC
-        ui
+        compiler_ui
 )
 
 target_include_directories(${PROJECT_NAME} PUBLIC ${PROJECT_SOURCE_DIR})

bin/main.cpp

@@ -1,8 +1,16 @@
 #include <iostream>
 
-#include "lib/ui/ui_functions.hpp"
+#include "lib/compiler_ui/compiler_ui_functions.hpp"
 
 int main(int32_t argc, char** argv) {
-  std::vector<std::string> args = std::vector<std::string>(argv, argv + argc);
-  return StartConsoleUI(args, std::cout);
+  const std::string sample = R"ovum(
+// demo
+fun Main(args: StringArray): Int {
+    val count: Int = args.Length()
+    sys::Print("Args count: " + count.ToString())
+    return 0
+}
+)ovum";
+
+  return StartCompilerConsoleUI({"ovumc", sample}, std::cout, std::cerr);
 }

cmake/SetCompilerOptions.cmake

@@ -1,4 +1,4 @@
-set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD 23)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 

lib/CMakeLists.txt

@@ -1,4 +1,2 @@
-cmake_minimum_required(VERSION 3.12)
-
-add_subdirectory(mylib)
-add_subdirectory(ui)
+add_subdirectory(lexer)
+add_subdirectory(compiler_ui)

lib/compiler_ui/CMakeLists.txt

@@ -0,0 +1,10 @@
+add_library(compiler_ui STATIC
+        compiler_ui_functions.cpp
+        compiler_ui_functions.hpp
+)
+
+target_link_libraries(compiler_ui PUBLIC
+        lexer
+)
+
+target_include_directories(compiler_ui PUBLIC ${PROJECT_SOURCE_DIR})

lib/compiler_ui/compiler_ui_functions.cpp

@@ -0,0 +1,28 @@
+#include "lib/lexer/Lexer.hpp"
+
+#include <exception>
+
+#include "compiler_ui_functions.hpp"
+
+int32_t StartCompilerConsoleUI(const std::vector<std::string>& args, std::ostream& out, std::ostream& err) {
+  if (args.size() < 2) {
+    err << "Insufficient arguments\n";
+    return 1;
+  }
+
+  const std::string& sample = args[1];
+  Lexer lx(sample, true);
+
+  try {
+    auto toks = lx.Tokenize();
+
+    for (auto& t : toks) {
+      out << t->ToString() << "\n";
+    }
+  } catch (const std::exception& e) {
+    err << "Lexer error: " << e.what() << "\n";
+    return 1;
+  }
+
+  return 0;
+}

lib/compiler_ui/compiler_ui_functions.hpp

@@ -0,0 +1,11 @@
+#ifndef COMPILER_UI_FUNCTIONS_HPP_
+#define COMPILER_UI_FUNCTIONS_HPP_
+
+#include <cstdint>
+#include <ostream>
+#include <string>
+#include <vector>
+
+int32_t StartCompilerConsoleUI(const std::vector<std::string>& args, std::ostream& out, std::ostream& err);
+
+#endif // COMPILER_UI_FUNCTIONS_HPP_

lib/lexer/CMakeLists.txt

@@ -0,0 +1,41 @@
+add_library(lexer STATIC
+        Lexer.cpp
+        SourceCodeWrapper.cpp
+        handlers/CharHandler.cpp
+        handlers/ColonHandler.cpp
+        handlers/DefaultHandler.cpp
+        handlers/DotCompositeHandler.cpp
+        handlers/IdentifierHandler.cpp
+        handlers/NewlineHandler.cpp
+        handlers/NumberHandler.cpp
+        handlers/OperatorHandler.cpp
+        handlers/PunctHandler.cpp
+        handlers/SlashHandler.cpp
+        handlers/StringHandler.cpp
+        handlers/WhitespaceHandler.cpp
+        tokens/CommentToken.cpp
+        tokens/EofToken.cpp
+        tokens/IdentToken.cpp
+        tokens/KeywordToken.cpp
+        tokens/LiteralToken.cpp
+        tokens/NewlineToken.cpp
+        tokens/OperatorToken.cpp
+        tokens/PunctToken.cpp
+        values/BoolValue.cpp
+        values/CharValue.cpp
+        values/FloatValue.cpp
+        values/IntValue.cpp
+        values/StringValue.cpp
+        tokens/TokenFactory.cpp
+        tokens/TokenPosition.cpp
+)
+
+target_include_directories(lexer
+        PUBLIC
+        ${CMAKE_SOURCE_DIR}
+)
+
+target_include_directories(lexer
+        PRIVATE
+        ${CMAKE_CURRENT_SOURCE_DIR}
+)

lib/lexer/Lexer.cpp

@@ -0,0 +1,107 @@
+#include "Lexer.hpp"
+
+#include <string>
+#include <utility>
+
+#include "handlers/CharHandler.hpp"
+#include "handlers/ColonHandler.hpp"
+#include "handlers/DefaultHandler.hpp"
+#include "handlers/DotCompositeHandler.hpp"
+#include "handlers/IdentifierHandler.hpp"
+#include "handlers/NewlineHandler.hpp"
+#include "handlers/NumberHandler.hpp"
+#include "handlers/OperatorHandler.hpp"
+#include "handlers/PunctHandler.hpp"
+#include "handlers/SlashHandler.hpp"
+#include "handlers/StringHandler.hpp"
+#include "handlers/WhitespaceHandler.hpp"
+
+#include "tokens/TokenFactory.hpp"
+
+namespace {
+constexpr const char* kOperatorChars = "+-*%<>=!&|^~?";
+constexpr const char* kPunctChars = ",;(){}[]";
+} // namespace
+
+Lexer::Lexer(std::string_view src, bool keep_comments) :
+    wrapper_(src, keep_comments), handlers_(MakeDefaultHandlers()), default_handler_(MakeDefaultHandler()) {
+}
+
+std::vector<TokenPtr> Lexer::Tokenize() {
+  std::vector<TokenPtr> tokens;
+  tokens.reserve(kDefaultTokenReserve);
+
+  while (!wrapper_.IsAtEnd()) {
+    wrapper_.ResetTokenPosition();
+
+    char ch_read = wrapper_.Advance();
+    Handler* current_handler = handlers_.at(static_cast<unsigned char>(ch_read)).get();
+
+    if (!current_handler) {
+      current_handler = default_handler_.get();
+    }
+
+    OptToken maybe_token = current_handler->Scan(wrapper_);
+
+    if (maybe_token && *maybe_token) {
+      tokens.push_back(std::move(*maybe_token));
+    }
+  }
+
+  tokens.push_back(TokenFactory::MakeEof(wrapper_.GetLine(), wrapper_.GetCol()));
+  return tokens;
+}
+
+void Lexer::SetHandler(unsigned char ch, std::unique_ptr<Handler> handler) {
+  handlers_.at(ch) = std::move(handler);
+}
+
+void Lexer::SetDefaultHandler(std::unique_ptr<Handler> handler) {
+  default_handler_ = std::move(handler);
+}
+
+std::array<std::unique_ptr<Handler>, kDefaultTokenReserve> Lexer::MakeDefaultHandlers() {
+  std::array<std::unique_ptr<Handler>, kDefaultTokenReserve> table{};
+  table.at(static_cast<unsigned char>(' ')) = std::make_unique<WhitespaceHandler>();
+  table.at(static_cast<unsigned char>('\t')) = std::make_unique<WhitespaceHandler>();
+  table.at(static_cast<unsigned char>('\r')) = std::make_unique<WhitespaceHandler>();
+
+  table.at(static_cast<unsigned char>('\n')) = std::make_unique<NewlineHandler>();
+
+  for (unsigned char ch = 'a'; ch <= 'z'; ++ch) {
+    table.at(ch) = std::make_unique<IdentifierHandler>();
+  }
+
+  for (unsigned char ch = 'A'; ch <= 'Z'; ++ch) {
+    table.at(ch) = std::make_unique<IdentifierHandler>();
+  }
+
+  table.at(static_cast<unsigned char>('_')) = std::make_unique<IdentifierHandler>();
+  table.at(static_cast<unsigned char>('#')) = std::make_unique<IdentifierHandler>();
+
+  for (unsigned char digit = '0'; digit <= '9'; ++digit) {
+    table.at(digit) = std::make_unique<NumberHandler>();
+  }
+
+  table.at(static_cast<unsigned char>('.')) = std::make_unique<DotCompositeHandler>();
+
+  table.at(static_cast<unsigned char>('"')) = std::make_unique<StringHandler>();
+  table.at(static_cast<unsigned char>('\'')) = std::make_unique<CharHandler>();
+  table.at(static_cast<unsigned char>('/')) = std::make_unique<SlashHandler>();
+
+  table.at(static_cast<unsigned char>(':')) = std::make_unique<ColonHandler>();
+
+  for (const unsigned char ch : std::string(kOperatorChars)) {
+    table.at(ch) = std::make_unique<OperatorHandler>();
+  }
+
+  for (const unsigned char ch : std::string(kPunctChars)) {
+    table.at(ch) = std::make_unique<PunctHandler>();
+  }
+
+  return table;
+}
+
+std::unique_ptr<Handler> Lexer::MakeDefaultHandler() {
+  return std::make_unique<DefaultHandler>();
+}

lib/lexer/Lexer.hpp

@@ -0,0 +1,35 @@
+#ifndef LEXER_HPP_
+#define LEXER_HPP_
+
+#include <array>
+#include <cstddef>
+#include <memory>
+#include <string_view>
+#include <vector>
+
+#include "SourceCodeWrapper.hpp"
+#include "handlers/Handler.hpp"
+
+constexpr std::size_t kDefaultTokenReserve = 256;
+
+class Lexer {
+public:
+  explicit Lexer(std::string_view src, bool keep_comments = false);
+
+  std::vector<TokenPtr> Tokenize();
+
+  void SetHandler(unsigned char ch, std::unique_ptr<Handler> handler);
+
+  void SetDefaultHandler(std::unique_ptr<Handler> handler);
+
+private:
+  static std::array<std::unique_ptr<Handler>, kDefaultTokenReserve> MakeDefaultHandlers();
+
+  static std::unique_ptr<Handler> MakeDefaultHandler();
+
+  SourceCodeWrapper wrapper_;
+  std::array<std::unique_ptr<Handler>, kDefaultTokenReserve> handlers_{};
+  std::unique_ptr<Handler> default_handler_;
+};
+
+#endif // LEXER_HPP_

lib/lexer/LexerError.hpp

@@ -0,0 +1,11 @@
+#ifndef LEXERERROR_HPP_
+#define LEXERERROR_HPP_
+
+#include <stdexcept>
+
+class LexerError : public std::runtime_error {
+public:
+  using std::runtime_error::runtime_error;
+};
+
+#endif // LEXERERROR_HPP_