shaderc: Start a rewrite of the shader compiler in the engine tree

engine/CMakeLists.txt

@@ -4,6 +4,7 @@ vull_module(json vull::core)
 vull_module(main vull::core)
 vull_module(physics vull::core)
 vull_module(script vull::core)
+vull_module(shaderc vull::core)
 vull_module(ui vull::core vull::vulkan PRIVATE Freetype::Freetype harfbuzz::harfbuzz)
 vull_module(vulkan vull::core)
 vull_module(x11-window vull::core PRIVATE X11::xcb X11::xcb_randr X11::xcb_util X11::xcb_xkb)
@@ -69,7 +70,7 @@ if(VULL_BUILD_TESTS)
     add_executable(vull-tests)
     add_subdirectory(tests)
     target_compile_features(vull-tests PRIVATE cxx_std_20)
-    target_link_libraries(vull-tests PRIVATE vull::core vull::json vull::script)
+    target_link_libraries(vull-tests PRIVATE vull::core vull::json vull::script vull::shaderc)
 
     add_custom_command(TARGET vull-tests
         POST_BUILD

engine/include/vull/shaderc/lexer.hh

@@ -0,0 +1,41 @@
+#pragma once
+
+#include <vull/shaderc/token.hh>
+#include <vull/support/lexer_base.hh>
+#include <vull/support/string.hh>
+#include <vull/support/string_view.hh>
+
+#include <stdint.h>
+
+namespace vull::shaderc {
+
+struct SourcePosition {
+    StringView file_name;
+    StringView line_source;
+    uint32_t line;
+    uint32_t column;
+};
+
+class Lexer : public LexerBase<Lexer, Token> {
+    friend LexerBase<Lexer, Token>;
+
+private:
+    String m_file_name;
+    String m_source;
+    uint32_t m_head{0};
+    uint16_t m_line{1};
+
+    static bool is_eof(const Token &token) { return token.kind() == TokenKind::Eof; }
+    void skip_char() { m_head++; }
+    void unskip_char() { m_head--; }
+    char peek_char() { return m_source[m_head]; }
+    char next_char() { return m_source[m_head++]; }
+    Token next_token();
+
+public:
+    Lexer(String file_name, String source);
+
+    SourcePosition recover_position(const Token &token) const;
+};
+
+} // namespace vull::shaderc

engine/include/vull/shaderc/token.hh

@@ -0,0 +1,65 @@
+#pragma once
+
+#include <vull/support/string.hh>
+#include <vull/support/string_view.hh>
+
+#include <stddef.h>
+
+namespace vull::shaderc {
+
+enum class TokenKind : uint16_t {
+    Invalid = 256,
+    Eof,
+    Identifier,
+    FloatLit,
+    IntLit,
+
+    PlusEqual,
+    MinusEqual,
+    AsteriskEqual,
+    SlashEqual,
+
+    KW_fn,
+    KW_let,
+    KW_pipeline,
+    KW_uniform,
+    KW_var,
+};
+
+class Token {
+    const void *m_ptr_data{nullptr};
+    union {
+        size_t int_data;
+        float float_data;
+    } m_number_data{};
+    uint32_t m_position;
+    uint16_t m_line;
+    TokenKind m_kind;
+
+public:
+    static String kind_string(TokenKind kind);
+
+    Token(TokenKind kind, uint32_t position, uint16_t line) : m_position(position), m_line(line), m_kind(kind) {}
+    Token(float decimal, uint32_t position, uint16_t line)
+        : m_number_data{.float_data = decimal}, m_position(position), m_line(line), m_kind(TokenKind::FloatLit) {}
+    Token(size_t integer, uint32_t position, uint16_t line)
+        : m_number_data{.int_data = integer}, m_position(position), m_line(line), m_kind(TokenKind::IntLit) {}
+    Token(TokenKind kind, StringView string, uint32_t position, uint16_t line)
+        : m_ptr_data(string.data()), m_number_data{.int_data = string.length()}, m_position(position), m_line(line),
+          m_kind(kind) {}
+
+    float decimal() const;
+    size_t integer() const;
+    StringView string() const;
+    String to_string() const;
+
+    TokenKind kind() const { return m_kind; }
+    uint32_t position() const { return m_position; }
+    uint16_t line() const { return m_line; }
+};
+
+constexpr TokenKind operator""_tk(char ch) {
+    return static_cast<TokenKind>(ch);
+}
+
+} // namespace vull::shaderc

engine/sources/CMakeLists.txt

@@ -54,6 +54,9 @@ target_sources(vull-script PRIVATE
     script/parser.cc
     script/vm.cc)
 
+target_sources(vull-shaderc PRIVATE
+    shaderc/lexer.cc)
+
 target_sources(vull-ui PRIVATE
     ui/layout/box_layout.cc
     ui/layout/pane.cc

engine/sources/shaderc/lexer.cc

@@ -0,0 +1,192 @@
+#include <vull/shaderc/lexer.hh>
+
+#include <vull/shaderc/token.hh>
+#include <vull/support/assert.hh>
+#include <vull/support/enum.hh>
+#include <vull/support/optional.hh>
+#include <vull/support/string.hh>
+#include <vull/support/string_builder.hh>
+#include <vull/support/string_view.hh>
+#include <vull/support/utility.hh>
+#include <vull/support/variant.hh>
+
+#include <stddef.h>
+
+// TODO: Deduplicate functionality between this lexer and the script lexer.
+
+#define MAKE_TOKEN(...)                                                                                                \
+    { __VA_ARGS__, position, m_line }
+
+namespace vull::shaderc {
+
+Lexer::Lexer(String file_name, String source) : m_file_name(vull::move(file_name)), m_source(vull::move(source)) {
+    if (m_source.empty()) {
+        m_peek_token.emplace(TokenKind::Eof, 0u, uint16_t(0));
+    }
+}
+
+Token Lexer::next_token() {
+    char ch;
+    while (is_space(ch = m_source[m_head++])) {
+        if (ch == '\n') {
+            m_line++;
+        }
+    }
+
+    const auto position = m_head - 1;
+    if (is_digit(ch)) {
+        auto number = parse_number(ch);
+        if (const auto decimal = number.try_get<double>()) {
+            return MAKE_TOKEN(static_cast<float>(*decimal));
+        }
+        return MAKE_TOKEN(number.get<uint64_t>());
+    }
+
+    if (is_ident(ch)) {
+        while (is_ident(m_source[m_head]) || is_digit(m_source[m_head])) {
+            m_head++;
+        }
+        auto string = m_source.view().substr(position, m_head);
+        if (string == "fn") {
+            return MAKE_TOKEN(TokenKind::KW_fn);
+        }
+        if (string == "let") {
+            return MAKE_TOKEN(TokenKind::KW_let);
+        }
+        if (string == "pipeline") {
+            return MAKE_TOKEN(TokenKind::KW_pipeline);
+        }
+        if (string == "uniform") {
+            return MAKE_TOKEN(TokenKind::KW_uniform);
+        }
+        if (string == "var") {
+            return MAKE_TOKEN(TokenKind::KW_var);
+        }
+        return MAKE_TOKEN(TokenKind::Identifier, string);
+    }
+
+    auto consume = [this](char ch) {
+        if (m_source[m_head] == ch) {
+            m_head++;
+            return true;
+        }
+        return false;
+    };
+
+    if (ch == '\0') {
+        m_peek_token.emplace(TokenKind::Eof, 0u, uint16_t(0));
+        return MAKE_TOKEN(TokenKind::Eof);
+    }
+
+    if (ch == '/' && consume('/')) {
+        while (m_source[m_head] != '\n' && m_source[m_head] != '\0') {
+            m_head++;
+        }
+        return next_token();
+    }
+
+    if (ch == '+' && consume('=')) {
+        return MAKE_TOKEN(TokenKind::PlusEqual);
+    }
+    if (ch == '-' && consume('=')) {
+        return MAKE_TOKEN(TokenKind::MinusEqual);
+    }
+    if (ch == '*' && consume('=')) {
+        return MAKE_TOKEN(TokenKind::AsteriskEqual);
+    }
+    if (ch == '/' && consume('=')) {
+        return MAKE_TOKEN(TokenKind::SlashEqual);
+    }
+
+    if (ch <= 31) {
+        return MAKE_TOKEN(TokenKind::Invalid);
+    }
+    return MAKE_TOKEN(static_cast<TokenKind>(ch));
+}
+
+SourcePosition Lexer::recover_position(const Token &token) const {
+    // Backtrack to find line start.
+    uint32_t line_head = token.position();
+    while (line_head > 1 && m_source[line_head - 1] != '\n') {
+        line_head--;
+    }
+
+    // Advance to find line end.
+    uint32_t line_end = token.position();
+    while (line_end < m_source.length() && m_source[line_end] != '\n') {
+        line_end++;
+    }
+
+    const auto line_view = m_source.view().substr(line_head, line_end);
+    return {m_file_name, line_view, token.line(), token.position() - line_head + 1};
+}
+
+float Token::decimal() const {
+    VULL_ASSERT(m_kind == TokenKind::FloatLit);
+    return m_number_data.float_data;
+}
+
+size_t Token::integer() const {
+    VULL_ASSERT(m_kind == TokenKind::IntLit);
+    return m_number_data.int_data;
+}
+
+StringView Token::string() const {
+    VULL_ASSERT(m_kind == TokenKind::Identifier);
+    return {static_cast<const char *>(m_ptr_data), m_number_data.int_data};
+}
+
+String Token::kind_string(TokenKind kind) {
+    if (auto value = vull::to_underlying(kind); value < 256) {
+        String string("'x'");
+        string.data()[1] = static_cast<char>(value);
+        return string;
+    }
+    switch (kind) {
+    case TokenKind::Invalid:
+        return "<invalid>";
+    case TokenKind::Eof:
+        return "<eof>";
+    case TokenKind::Identifier:
+        return "identifier";
+    case TokenKind::FloatLit:
+        return "float literal";
+    case TokenKind::IntLit:
+        return "integer literal";
+    case TokenKind::PlusEqual:
+        return "'+='";
+    case TokenKind::MinusEqual:
+        return "'-='";
+    case TokenKind::AsteriskEqual:
+        return "'*='";
+    case TokenKind::SlashEqual:
+        return "'/='";
+    case TokenKind::KW_fn:
+        return "'fn'";
+    case TokenKind::KW_let:
+        return "'let'";
+    case TokenKind::KW_pipeline:
+        return "'pipeline'";
+    case TokenKind::KW_uniform:
+        return "'uniform'";
+    case TokenKind::KW_var:
+        return "'var'";
+    default:
+        VULL_ENSURE_NOT_REACHED();
+    }
+}
+
+String Token::to_string() const {
+    switch (m_kind) {
+    case TokenKind::Identifier:
+        return vull::format("'{}'", string());
+    case TokenKind::FloatLit:
+        return vull::format("'{}f'", decimal());
+    case TokenKind::IntLit:
+        return vull::format("'{}u'", integer());
+    default:
+        return kind_string(m_kind);
+    }
+}
+
+} // namespace vull::shaderc

engine/tests/CMakeLists.txt

@@ -9,6 +9,7 @@ target_sources(vull-tests PRIVATE
     maths/epsilon.cc
     maths/relational.cc
     script/lexer.cc
+    shaderc/lexer.cc
     support/enum.cc
     support/variant.cc
     ui/units.cc