diff --git a/CMakeLists.txt b/CMakeLists.txt index 43281dee2c96c0..980e0dde1e7c64 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,8 +22,10 @@ set_property(GLOBAL PROPERTY USE_FOLDERS ON) # Prefer our bundled versions of dependencies. if(DEFINED ENV{DEPS_BUILD_DIR}) + set(DEPS_BUILD_DIR "$ENV{DEPS_BUILD_DIR") set(DEPS_PREFIX "$ENV{DEPS_BUILD_DIR}/usr" CACHE PATH "Path prefix for finding dependencies") else() + set(DEPS_BUILD_DIR "${CMAKE_CURRENT_SOURCE_DIR}/.deps") set(DEPS_PREFIX "${CMAKE_CURRENT_SOURCE_DIR}/.deps/usr" CACHE PATH "Path prefix for finding dependencies") # When running from within CLion or Visual Studio, # build bundled dependencies automatically. @@ -415,6 +417,11 @@ if(MSGPACK_HAS_FLOAT32) add_definitions(-DNVIM_MSGPACK_HAS_FLOAT32) endif() +set(TREESITTER_PATH "${DEPS_BUILD_DIR}/build/src/treesitter") +set(TREESITTER_C_PATH "${DEPS_BUILD_DIR}/build/src/treesitter-c") +set(TREESITTER_JAVASCRIPT_PATH "${DEPS_BUILD_DIR}/build/src/treesitter-javascript") +set(LIBUTF8PROC_INCLUDE_DIRS ${DEPS_BUILD_DIR}/build/src/utf8proc) + option(FEAT_TUI "Enable the Terminal UI" ON) if(FEAT_TUI) diff --git a/runtime/lua/treesitter_rt.lua b/runtime/lua/treesitter_rt.lua new file mode 100644 index 00000000000000..f692321b9d7539 --- /dev/null +++ b/runtime/lua/treesitter_rt.lua @@ -0,0 +1,321 @@ +_G.a = vim.api +local a = vim.api + +if __treesitter_rt_ns == nil then + __treesitter_rt_ns = a.nvim_buf_add_highlight(0, 0, "", 0, 0, 0) + __treesitter_rt_syn_ns = a.nvim_buf_add_highlight(0, 0, "", 0, 0, 0) +end +local my_ns = __treesitter_rt_ns +local my_syn_ns = __treesitter_rt_syn_ns + +function js_sheet() + local path = '.deps/build/src/treesitter-javascript/src/highlights.json' + a.nvim_set_var("_ts_path", path) + local obj = a.nvim_eval("json_decode(readfile(g:_ts_path,'b'))") + --for k in pairs(obj) do print(k) end + --obj.property_sets[2] + + states = obj.states + s = states[1] + --for k in pairs(s) do print(k) end + + t = s.transitions[2] + --for k in pairs(t) do print(k) end + + parser = vim.ts_parser("javascript") + symbs = parser:symbols() + named = {} + anonymous = {} + for i, symb in pairs(symbs) do + local dict + if symb[2] == "named" then + dict = named + --named[symb[1]] = i + elseif symb[2] == "anonymous" then + dict = anonymous + --anonymous[symb[1]] = i + else + dict = {} -- SKRAPET + end + -- TODO: duplicate symbols might be a bug + if dict[symb[1]] == nil then + dict[symb[1]] = {} + end + table.insert(dict[symb[1]], i) + end + lut = {[true]=named, [false]=anonymous} + + local sheet = vim.ts_propertysheet(#states, #symbs) + for _, s in pairs(states) do + local id = s.id + sheet:add_state(id, s.default_next_state_id, s.property_set_id) + for _,t in pairs(s.transitions) do + if t.text == nil then + local kinds = lut[t.named][t.type] + for _,kind in ipairs(kinds) do + sheet:add_transition(id, kind, t.state_id, t.index) + end + end + end + end + + scope = {} + for i,prop in ipairs(obj.property_sets) do + scope[i-1] = prop.scope + end + return sheet +end + +--luadev = require'luadev' +--i = require'inspect' + + +function reader(payload, byte_index, position, bytes_read) + val, status = pcall(function () + end) + return "" +end + +--print(read_cb(reader)) + +function parse_tree(tsstate, force) + if tsstate.valid and not force then + return tsstate.tree + end + tsstate.tree = tsstate.parser:parse_buf(tsstate.bufnr) + tsstate.valid = true + return tsstate.tree +end + +function the_cb(tsstate, ev, bufnr, tick, start_row, oldstopline, stop_row) + local start_byte = a.nvim_buf_get_offset(bufnr,start_row) + -- a bit messy, should we expose edited but not reparsed tree? + -- are multiple edits safe in general? + local root = tsstate.parser:tree():root() + -- TODO: add proper lookup function! + local inode = root:descendant_for_point_range(oldstopline+9000,0, oldstopline,0) + local edit + if inode == nil then + local stop_byte = a.nvim_buf_get_offset(bufnr,stop_row) + tsstate.parser:edit(start_byte,stop_byte,stop_byte,start_row,0,stop_row,0,stop_row,0) + else + local fakeoldstoprow, fakeoldstopcol, fakebyteoldstop = inode:start() + local fake_rows = fakeoldstoprow-oldstopline + local fakestop = stop_row+fake_rows + local fakebytestop = a.nvim_buf_get_offset(bufnr,fakestop)+fakeoldstopcol + tsstate.parser:edit(start_byte,fakebyteoldstop,fakebytestop,start_row,0,fakeoldstoprow,fakeoldstopcol,fakestop,fakeoldstopcol) + end + tsstate.valid = false + --luadev.append_buf({i{edit.start_byte,edit.old_end_byte,edit.new_end_byte}, + -- i{edit.start_point, edit.old_end_point, edit.new_end_point}}) +end + +function attach_buf(tsstate) + local function cb(ev, ...) + return the_cb(tsstate, ev, ...) + end + a.nvim_buf_attach(tsstate.bufnr, false, {on_lines=cb}) +end + +function create_parser(bufnr) + if bufnr == 0 then + bufnr = a.nvim_get_current_buf() + end + local ft = a.nvim_buf_get_option(bufnr, "filetype") + local tsstate = {} + tsstate.bufnr = bufnr + tsstate.parser = vim.ts_parser(ft) + parse_tree(tsstate) + attach_buf(tsstate) + return tsstate +end + +function ts_inspect_pos(row,col) + local tree = parse_tree(theparser) + local root = tree:root() + local node = root:descendant_for_point_range(row,col,row,col) + show_node(node) +end + +sheet = js_sheet() +function ts_inspect2(row,col) + local tree = parse_tree(theparser) + icursor = tree:root():to_cursor(sheet) + local startbyte = a.nvim_buf_get_offset(theparser.bufnr, row) + ipos = startbyte+col+1 + ii = 0 + repeat + node, propid = icursor:forward(ipos) + r,c, start_byte = node:start() + ii = ii + 1 + until propid > 0 or start_byte > ipos + show_node(node,true) + print(ii, scope[propid], node:type()) +end + +function ts_iforward() + node, propid = icursor:forward(ipos) + show_node(node,true) + print(node:type(), scope[propid]) +end + +function show_node(node,subtle) + if node == nil then + return + end + a.nvim_buf_clear_highlight(0, my_ns, 0, -1) + shown_node = node + if not subtle then + print(node:type()) + end + local start_row, start_col, end_row, end_col = node:range() + + a.nvim_buf_add_highlight(0, my_ns, "ErrorMsg", start_row, start_col, start_col+1) + + if end_col >= 1 then + end_col = end_col - 1 + end + a.nvim_buf_add_highlight(0, my_ns, "ErrorMsg", end_row, end_col, end_col+1) +end + +function ts_expand_node() + if shown_node == nil then + return + end + parent = shown_node:parent() + show_node(parent) +end + +function ts_cursor() + local row, col = unpack(a.nvim_win_get_cursor(0)) + ts_inspect_pos(row-1, col) + --ts_inspect2(row-1, col) +end + +hl_map = { + primitive_type="Type", + type_identifier="Identifier", + const="Type", + struct="Type", + typedef="Type", + enum="Type", + static="Type", + ["if"]="Statement", + ["for"]="Statement", + ["while"]="Statement", + ["return"]="Statement", + number_literal="Number", + string_literal="String", + comment="Comment", + ["#include"]="PreProc", + ["#define"]="PreProc", + ["#ifdef"]="PreProc", + ["#else"]="PreProc", + ["#endif"]="PreProc", +} + +hl_scope_map = { + constant='Constant', + number='Number', + keyword='Statement', + string='String', + escape='Special', + comment='Comment', + ['function']='Identifier', +} + +id_map = {} +for k,v in pairs(hl_map) do + id_map[k] = a.nvim__syn_attr(v) +end + +scope_map = {} +id_scope_map = {} +for i,s in pairs(scope) do + if hl_scope_map[s] then + scope_map[i] = hl_scope_map[s] + id_scope_map[i] = a.nvim__syn_attr(hl_scope_map[s]) + end +end + + +function ts_line(line,endl,drawing) + if endl == nil then endl = line+1 end + if not drawing then + a.nvim_buf_clear_highlight(0, my_syn_ns, line, endl) + end + tree = parse_tree(theparser) + local root = tree:root() + local cursor = root:to_cursor(sheet) + print(cursor) + local startbyte = a.nvim_buf_get_offset(theparser.bufnr, line) + local node = root + local continue = true + local i = 800 + local nscope = 0 + while continue do + --print(inspect_node(node)) + if true then + local map = (drawing and id_scope_map) or scope_map + hl = map[nscope] + else + local name = node:type() + local map = (drawing and id_map) or hl_map + local hl = map[name] + end + local start_row, start_col, end_row, end_col = node:range() + if hl then + if not drawing then + --print(inspect_node(node)) + print(hl) + end + if start_row == end_row then + if drawing then + if start_row == line then + a.nvim__put_attr(hl, start_col, end_col) + end + else + a.nvim_buf_add_highlight(theparser.bufnr, my_syn_ns, hl, start_row, start_col, end_col) + end + end + end + if start_row >= endl then + continue = false + end + node, nscope = cursor:forward(startbyte) + if node == nil then + continue = false + end + + i = i - 1 + if i == 0 then + continue = false + end + end +end + +if false then + ts_line(0,800) +end + + +function ts_on_winhl(win, buf, lnum) + ts_line(lnum, lnum+1, true) +end + +function ts_syntax() + a.nvim_buf_set_luahl(theparser.bufnr, "return ts_on_winhl(...)") +end + +if false then + ctree = theparser.tree + root = ctree:root() + cursor = root:to_cursor() + node = cursor:forward(5000) if true then return node end + print(#root) + c = root:child(50) + print(require'inspect'{c:extent()}) + type(ctree.__tostring) + root:__tostring() + print(_tslua_debug()) +end diff --git a/runtime/plugin/ts_test.vim b/runtime/plugin/ts_test.vim new file mode 100644 index 00000000000000..5527802e5c1c50 --- /dev/null +++ b/runtime/plugin/ts_test.vim @@ -0,0 +1,33 @@ +let g:ts_test_path = expand(":p:h:h") +let g:has_ts = v:false + +func! TSTest() + if g:has_ts + return + end + " TODO: module! + lua require'treesitter_rt' + lua theparser = create_parser(vim.api.nvim_get_current_buf()) + let g:has_ts = v:true +endfunc + +func! TSCursor() + " disable matchparen + NoMatchParen + call TSTest() + au CursorMoved lua ts_cursor() + au CursorMovedI lua ts_cursor() + map (ts-expand) lua ts_expand_node() + map (ts-fwd) lua ts_iforward() +endfunc + +func! TSSyntax() + " disable matchparen + set syntax= + call TSTest() + lua ts_syntax() +endfunc + +command! TSTest call TSTest() +command! TSCursor call TSCursor() +command! TSSyntax call TSSyntax() diff --git a/src/nvim/CMakeLists.txt b/src/nvim/CMakeLists.txt index 8abc43c2aa311d..0ed732a7155781 100644 --- a/src/nvim/CMakeLists.txt +++ b/src/nvim/CMakeLists.txt @@ -72,6 +72,10 @@ include_directories(${GENERATED_DIR}) include_directories(${CACHED_GENERATED_DIR}) include_directories(${GENERATED_INCLUDES_DIR}) +include_directories(${TREESITTER_PATH}/lib/include) +include_directories(${TREESITTER_PATH}/lib/src) +include_directories(${LIBUTF8PROC_INCLUDE_DIRS}) + file(MAKE_DIRECTORY ${TOUCHES_DIR}) file(MAKE_DIRECTORY ${GENERATED_DIR}) file(MAKE_DIRECTORY ${GENERATED_INCLUDES_DIR}) @@ -83,6 +87,11 @@ file(GLOB NVIM_HEADERS *.h) file(GLOB XDIFF_SOURCES xdiff/*.c) file(GLOB XDIFF_HEADERS xdiff/*.h) +list(APPEND TS_SOURCES ${TREESITTER_PATH}/lib/src/lib.c) +list(APPEND TS_SOURCES ${TREESITTER_C_PATH}/src/parser.c) +list(APPEND TS_SOURCES ${TREESITTER_JAVASCRIPT_PATH}/src/parser.c) +list(APPEND TS_SOURCES ${TREESITTER_JAVASCRIPT_PATH}/src/scanner.c) + foreach(subdir os api @@ -140,6 +149,7 @@ set(CONV_SOURCES ex_docmd.c ex_getln.c fileio.c + lua/tree_sitter.c mbyte.c memline.c message.c @@ -157,7 +167,7 @@ foreach(sfile ${CONV_SOURCES}) endif() endforeach() # xdiff: inlined external project, we don't maintain it. #9306 -list(APPEND CONV_SOURCES ${XDIFF_SOURCES}) +list(APPEND CONV_SOURCES ${XDIFF_SOURCES} ${TS_SOURCES}) if(NOT MSVC) set_source_files_properties( @@ -406,7 +416,7 @@ endif() add_executable(nvim ${NVIM_GENERATED_FOR_SOURCES} ${NVIM_GENERATED_FOR_HEADERS} ${NVIM_GENERATED_SOURCES} ${NVIM_SOURCES} ${NVIM_HEADERS} - ${XDIFF_SOURCES} ${XDIFF_HEADERS}) + ${XDIFF_SOURCES} ${XDIFF_HEADERS} ${TS_SOURCES}) target_link_libraries(nvim ${NVIM_EXEC_LINK_LIBRARIES}) install_helper(TARGETS nvim) @@ -492,7 +502,7 @@ add_library( EXCLUDE_FROM_ALL ${NVIM_SOURCES} ${NVIM_GENERATED_SOURCES} ${NVIM_HEADERS} ${NVIM_GENERATED_FOR_SOURCES} ${NVIM_GENERATED_FOR_HEADERS} - ${XDIFF_SOURCES} ${XDIFF_HEADERS} + ${XDIFF_SOURCES} ${XDIFF_HEADERS} ${TS_SOURCES} ) set_property(TARGET libnvim APPEND PROPERTY INCLUDE_DIRECTORIES ${LUA_PREFERRED_INCLUDE_DIRS}) @@ -517,7 +527,7 @@ else() EXCLUDE_FROM_ALL ${NVIM_SOURCES} ${NVIM_GENERATED_SOURCES} ${NVIM_HEADERS} ${NVIM_GENERATED_FOR_SOURCES} ${NVIM_GENERATED_FOR_HEADERS} - ${XDIFF_SOURCES} ${XDIFF_HEADERS} + ${XDIFF_SOURCES} ${XDIFF_HEADERS} ${TS_SOURCES} ${UNIT_TEST_FIXTURES} ) target_link_libraries(nvim-test ${NVIM_TEST_LINK_LIBRARIES}) diff --git a/src/nvim/api/buffer.c b/src/nvim/api/buffer.c index 81b3851c53352c..a251c8ae4d8032 100644 --- a/src/nvim/api/buffer.c +++ b/src/nvim/api/buffer.c @@ -179,6 +179,23 @@ Boolean nvim_buf_detach(uint64_t channel_id, return true; } +/// TODO: use ref +void nvim_buf_set_luahl(Buffer buffer, String cb, Error *err) + FUNC_API_SINCE(5) +{ + buf_T *buf = find_buffer_by_handle(buffer, err); + + if (!buf) { + return; + } + xfree(buf->b_luahl); + if (cb.size > 0) { + buf->b_luahl = xstrdup(cb.data); + } else { + buf->b_luahl = NULL; + } +} + /// Sets a buffer line /// /// @deprecated use nvim_buf_set_lines instead. diff --git a/src/nvim/api/vim.c b/src/nvim/api/vim.c index 2e8ca384b4cf25..ba491557e422ef 100644 --- a/src/nvim/api/vim.c +++ b/src/nvim/api/vim.c @@ -2256,3 +2256,26 @@ Array nvim__inspect_cell(Integer row, Integer col, Error *err) } return ret; } + +Integer nvim__syn_attr(String name) +{ + int id = syn_name2id((const char_u *)name.data); + if (id == 0 || syn_get_final_id(id) == 0) { + return 0; + } + int attrcode = syn_id2attr(id); + return attrcode; +} + +void nvim__put_attr(Integer id, Integer c0, Integer c1) +{ + if (!lua_attr_active) { + return; + } + c0 = MAX(c0, 0); + c1 = MIN(c1, (Integer)lua_attr_bufsize); + for (int c = (int)c0; c < (int)c1; c++) { + lua_attr_buf[c] = (sattr_T)hl_combine_attr(lua_attr_buf[c], (int)id); + } + return; +} diff --git a/src/nvim/buffer.c b/src/nvim/buffer.c index 52dc3597167bc8..2381b83ef19ae7 100644 --- a/src/nvim/buffer.c +++ b/src/nvim/buffer.c @@ -5738,3 +5738,38 @@ void buf_open_scratch(handle_T bufnr, char *bufname) set_option_value("swf", 0L, NULL, OPT_LOCAL); RESET_BINDING(curwin); } + + +/* +typedef struct { + uint32_t row; + uint32_t column; +} TSPoint; +*/ + +void *nvim_ts_read_payload(int bufnr) +{ + buf_T *bp = handle_get_buffer(bufnr); + return bp; +} + +const char *nvim_ts_read_cb(void *payload, uint32_t byte_index, TSPoint position, uint32_t *bytes_read) { + buf_T *bp = payload; + static char buf[200]; + if (position.row >= bp->b_ml.ml_line_count) { + *bytes_read = 0; + return ""; + } + char_u *line = ml_get_buf(bp, position.row+1, false); + size_t len = STRLEN(line); + size_t tocopy = MIN(len-position.column,200); + + // TODO: translate embedded \n to \000 + memcpy(buf, line+position.column, tocopy); + *bytes_read = (uint32_t)tocopy; + if (tocopy < 200) { + buf[tocopy] = '\n'; + (*bytes_read)++; + } + return buf; + } diff --git a/src/nvim/buffer.h b/src/nvim/buffer.h index ee3fda5f6dd2e8..02f713e5ea75f0 100644 --- a/src/nvim/buffer.h +++ b/src/nvim/buffer.h @@ -1,6 +1,8 @@ #ifndef NVIM_BUFFER_H #define NVIM_BUFFER_H +#include // for TSPoint + #include "nvim/vim.h" #include "nvim/window.h" #include "nvim/pos.h" // for linenr_T diff --git a/src/nvim/buffer_defs.h b/src/nvim/buffer_defs.h index 117a9183a40129..686773d75466ea 100644 --- a/src/nvim/buffer_defs.h +++ b/src/nvim/buffer_defs.h @@ -804,6 +804,8 @@ struct file_buffer { kvec_t(uint64_t) update_channels; kvec_t(BufUpdateCallbacks) update_callbacks; + char *b_luahl; + int b_diff_failed; // internal diff failed for this buffer }; diff --git a/src/nvim/globals.h b/src/nvim/globals.h index 78409267a42d30..d72c8736b2075d 100644 --- a/src/nvim/globals.h +++ b/src/nvim/globals.h @@ -135,6 +135,12 @@ typedef off_t off_T; */ EXTERN int mod_mask INIT(= 0x0); /* current key modifiers */ + +// TODO(bfredl): not here!! +EXTERN sattr_T *lua_attr_buf INIT(= NULL); +EXTERN size_t lua_attr_bufsize INIT(= 0); +EXTERN bool lua_attr_active INIT(= false); + /* * Cmdline_row is the row where the command line starts, just below the * last window. diff --git a/src/nvim/lua/executor.c b/src/nvim/lua/executor.c index df08a9dd8703b9..d6da97ef9fe03b 100644 --- a/src/nvim/lua/executor.c +++ b/src/nvim/lua/executor.c @@ -30,6 +30,7 @@ #include "nvim/lua/executor.h" #include "nvim/lua/converter.h" +#include "nvim/lua/tree_sitter.h" typedef struct { Error err; @@ -176,7 +177,10 @@ static int nlua_state_init(lua_State *const lstate) FUNC_ATTR_NONNULL_ALL lua_pushcfunction(lstate, &nlua_schedule); lua_setfield(lstate, -2, "schedule"); + nlua_add_treesitter(lstate); + lua_setglobal(lstate, "vim"); + return 0; } @@ -652,3 +656,42 @@ void ex_luafile(exarg_T *const eap) return; } } + +static int create_tslua_parser(lua_State *L) +{ + TSLanguage *tree_sitter_c(void), *tree_sitter_javascript(void); + + if (!lua_gettop(L)) { + return 0; + } + char *str = lua_tostring(L,1); + + TSLanguage *lang = tree_sitter_c(); + if (str && striequal(str, "javascript")) { + lang = tree_sitter_javascript(); + } + tslua_push_parser(L, lang); + return 1; +} + +static int create_tslua_propertysheet(lua_State *L) +{ + if (lua_gettop(L) < 2) { + return 0; + } + int n_states = lua_tointeger(L,1); + int n_kinds = lua_tointeger(L,2); + + tslua_push_propertysheet(L, n_states, n_kinds); + return 1; +} +static void nlua_add_treesitter(lua_State *const lstate) FUNC_ATTR_NONNULL_ALL +{ + tslua_init(lstate); + + lua_pushcfunction(lstate, create_tslua_parser); + lua_setfield(lstate, -2, "ts_parser"); + + lua_pushcfunction(lstate, create_tslua_propertysheet); + lua_setfield(lstate, -2, "ts_propertysheet"); +} diff --git a/src/nvim/lua/tree_sitter.c b/src/nvim/lua/tree_sitter.c new file mode 100644 index 00000000000000..88557067af403b --- /dev/null +++ b/src/nvim/lua/tree_sitter.c @@ -0,0 +1,880 @@ +// This is an open source non-commercial project. Dear PVS-Studio, please check +// it. PVS-Studio Static Code Analyzer for C, C++ and C#: http://www.viva64.com + +// lua bindings for tree-siter. +// NB: this file should contain a generic lua interface for +// tree-sitter trees and nodes, and could be broken out as a reusable library + +#include +#include +#include +#include + +#include +#include +#include + +#include + +#define REG_KEY "tree_sitter-private" + +#include "nvim/lib/kvec.h" +#include "nvim/lua/tree_sitter.h" +#include "nvim/buffer.h" // for nvim_ts_read_cb + +typedef struct { + TSParser *parser; + TSTree *tree; +} Tslua_parser; + +typedef struct { + int kind_id; + int next_state_id; + int child_index; + int regex_index; +} KindTransition; + +typedef struct { + int default_next_state_id; + int property_id; + kvec_t(KindTransition) kind_trans; + int *kind_first_trans; +} PropertyState; + +typedef struct { + PropertyState *states; + int n_states; + int n_kinds; +} Tslua_propertysheet; + +typedef struct { + TSTreeCursor cursor; + Tslua_propertysheet *sheet; + int state_id[32]; + int child_index[32]; + int level; + //Buffer source; +} Tslua_cursor; + +#ifdef INCLUDE_GENERATED_DECLARATIONS +# include "lua/tree_sitter.c.generated.h" +#endif + +static struct luaL_Reg parser_meta[] = { + {"__gc", parser_gc}, + {"__tostring", parser_tostring}, + {"parse_buf", parser_parse_buf}, + {"edit", parser_edit}, + {"tree", parser_tree}, + {"symbols", parser_symbols}, + {NULL, NULL} +}; + +static struct luaL_Reg tree_meta[] = { + {"__gc", tree_gc}, + {"__tostring", tree_tostring}, + {"root", tree_root}, + {NULL, NULL} +}; + +static struct luaL_Reg node_meta[] = { + {"__tostring", node_tostring}, + {"__len", node_child_count}, + {"range", node_range}, + {"start", node_start}, + {"end_byte", node_end_byte}, + {"type", node_type}, + {"symbol", node_symbol}, + {"child_count", node_child_count}, + {"child", node_child}, + {"descendant_for_point_range", node_descendant_for_point_range}, + {"parent", node_parent}, + {"to_cursor", node_to_cursor}, + {NULL, NULL} +}; + +static struct luaL_Reg cursor_meta[] = { + {"__gc", cursor_gc}, + {"__tostring", cursor_tostring}, + //{"node", cursor_node}, + {"forward", cursor_forward}, + {"debug", cursor_debug}, + {NULL, NULL} +}; + +static struct luaL_Reg propertysheet_meta[] = { + {"__gc", propertysheet_gc}, + {"__tostring", propertysheet_tostring}, + {"add_state", propertysheet_add_state}, + {"add_transition", propertysheet_add_transition}, + {NULL, NULL} +}; + + +void build_meta(lua_State *L, const luaL_Reg *meta) +{ + // [env, target] + for (size_t i = 0; meta[i].name != NULL; i++) { + lua_pushcfunction(L, meta[i].func); // [env, target, func] + lua_pushvalue(L, -3); // [env, target, func, env] + lua_setfenv(L, -2); // [env, target, func] + lua_setfield(L, -2, meta[i].name); // [env, target] + } + + lua_pushvalue(L, -1); // [env, target, target] + lua_setfield(L, -2, "__index"); // [env, target] +} + + + +/// init the tslua library +/// +/// all global state is stored in the regirstry of the lua_State +void tslua_init(lua_State *L) +{ + lua_createtable(L, 0, 0); + + // type metatables + lua_createtable(L, 0, 0); + build_meta(L, parser_meta); + lua_setfield(L, -2, "parser-meta"); + + lua_createtable(L, 0, 0); + build_meta(L, tree_meta); + lua_setfield(L, -2, "tree-meta"); + + lua_createtable(L, 0, 0); + build_meta(L, node_meta); + lua_setfield(L, -2, "node-meta"); + + lua_createtable(L, 0, 0); + build_meta(L, cursor_meta); + lua_setfield(L, -2, "cursor-meta"); + + lua_createtable(L, 0, 0); + build_meta(L, propertysheet_meta); + lua_setfield(L, -2, "propertysheet-meta"); + + lua_setfield(L, LUA_REGISTRYINDEX, REG_KEY); +} + +void tslua_push_parser(lua_State *L, TSLanguage *lang) +{ + TSParser *parser = ts_parser_new(); + ts_parser_set_language(parser, lang); + Tslua_parser *p = lua_newuserdata(L, sizeof(Tslua_parser)); // [udata] + p->parser = parser; + p->tree = NULL; + + lua_getfield(L, LUA_REGISTRYINDEX, REG_KEY); // [udata, env] + lua_getfield(L, -1, "parser-meta"); // [udata, env, meta] + lua_setmetatable(L, -3); // [udata, env] + lua_pop(L, 1); // [udata] +} + +static Tslua_parser *parser_check(lua_State *L) +{ + if (!lua_gettop(L)) { + return 0; + } + if (!lua_isuserdata(L, 1)) { + return 0; + } + // TODO: typecheck! + return lua_touserdata(L, 1); +} + +static int parser_gc(lua_State *L) +{ + Tslua_parser *p = parser_check(L); + if (!p) { + return 0; + } + + ts_parser_delete(p->parser); + if (p->tree) { + ts_tree_delete(p->tree); + } + + return 0; +} + +static int parser_tostring(lua_State *L) +{ + lua_pushstring(L, ""); + return 1; +} + +static int parser_parse_buf(lua_State *L) +{ + Tslua_parser *p = parser_check(L); + if (!p) { + return 0; + } + + long num = lua_tointeger(L, 2); + void *payload = nvim_ts_read_payload(num); + TSInput input = {payload, nvim_ts_read_cb, TSInputEncodingUTF8}; + TSTree *new_tree = ts_parser_parse(p->parser, p->tree, input); + if (p->tree) { + ts_tree_delete(p->tree); + } + p->tree = new_tree; + + tslua_push_tree(L, ts_tree_copy(p->tree)); + return 1; +} + +static int parser_tree(lua_State *L) +{ + Tslua_parser *p = parser_check(L); + if (!p) { + return 0; + } + + if (p->tree) { + tslua_push_tree(L, ts_tree_copy(p->tree)); + } else { + lua_pushnil(L); + } + return 1; +} + +static int parser_edit(lua_State *L) +{ + if(lua_gettop(L) < 10) { + lua_pushstring(L, "not enough args to parser:edit()"); + return lua_error(L); + } + + Tslua_parser *p = parser_check(L); + if (!p) { + return 0; + } + + if (!p->tree) { + return 0; + } + + long start_byte = lua_tointeger(L, 2); + long old_end_byte = lua_tointeger(L, 3); + long new_end_byte = lua_tointeger(L, 4); + TSPoint start_point = { lua_tointeger(L, 5), lua_tointeger(L, 6) }; + TSPoint old_end_point = { lua_tointeger(L, 7), lua_tointeger(L, 8) }; + TSPoint new_end_point = { lua_tointeger(L, 9), lua_tointeger(L, 10) }; + + TSInputEdit edit = { start_byte, old_end_byte, new_end_byte, + start_point, old_end_point, new_end_point }; + + ts_tree_edit(p->tree, &edit); + + return 0; +} + +static int parser_symbols(lua_State *L) +{ + Tslua_parser *p = parser_check(L); // [parser] + if (!p) { + return 0; + } + + const TSLanguage *lang = ts_parser_language(p->parser); + + size_t nsymb = (size_t)ts_language_symbol_count(lang); + + lua_createtable(L, nsymb-1, 1); // [parser, result] + for (size_t i = 0; i < nsymb; i++) { + lua_createtable(L, 2, 0); // [parser, result, elem] + lua_pushstring(L, ts_language_symbol_name(lang, i)); + lua_rawseti(L, -2, 1); + TSSymbolType t= ts_language_symbol_type(lang, i); + lua_pushstring(L, (t == TSSymbolTypeRegular + ? "named" : (t == TSSymbolTypeAnonymous + ? "anonymous" : "auxiliary"))); + lua_rawseti(L, -2, 2); // [parser, result, elem] + lua_rawseti(L, -2, i); // [parser, result] + } + + return 1; +} + + +// Tree methods + +/// push tree interface on lua stack. +/// +/// This takes "ownership" of the tree and will free it +/// when the wrapper object is garbage collected +void tslua_push_tree(lua_State *L, TSTree *tree) +{ + TSTree **ud = lua_newuserdata(L, sizeof(TSTree *)); // [udata] + *ud = tree; + lua_getfield(L, LUA_REGISTRYINDEX, REG_KEY); // [udata, env] + lua_getfield(L, -1, "tree-meta"); // [udata, env, meta] + lua_setmetatable(L, -3); // [udata, env] + lua_pop(L, 1); // [udata] + + // table used for node wrappers to keep a reference to tree wrapper + // NB: in lua 5.3 the uservalue for the node could just be the tree, but + // in lua 5.1 the uservalue (fenv) must be a table. + lua_createtable(L, 1, 0); // [udata, reftable] + lua_pushvalue(L, -2); // [udata, reftable, udata] + lua_rawseti(L, -2, 1); // [udata, reftable] + lua_setfenv(L, -2); // [udata] +} + +static TSTree *tree_check(lua_State *L) +{ + if (!lua_gettop(L)) { + return 0; + } + if (!lua_isuserdata(L, 1)) { + return 0; + } + // TODO: typecheck! + TSTree **ud = lua_touserdata(L, 1); + return *ud; +} + +static int tree_gc(lua_State *L) +{ + TSTree *tree = tree_check(L); + if (!tree) { + return 0; + } + + // ts_tree_delete(tree); + return 0; +} + +static int tree_tostring(lua_State *L) +{ + lua_pushstring(L, ""); + return 1; +} + +static int tree_root(lua_State *L) +{ + TSTree *tree = tree_check(L); + if (!tree) { + return 0; + } + TSNode root = ts_tree_root_node(tree); + push_node(L, root); + return 1; +} + +// Node methods + +/// push node interface on lua stack +/// +/// top of stack must either be the tree this node belongs to or another node +/// of the same tree! This value is not popped. Can only be called inside a +/// cfunction with the tslua environment. +static void push_node(lua_State *L, TSNode node) +{ + if (ts_node_is_null(node)) { + lua_pushnil(L); // [src, nil] + return; + } + TSNode *ud = lua_newuserdata(L, sizeof(TSNode)); // [src, udata] + *ud = node; + lua_getfield(L, LUA_ENVIRONINDEX, "node-meta"); // [src, udata, meta] + lua_setmetatable(L, -2); // [src, udata] + //lua_getfenv(L, -2); // [src, udata, reftable] + //lua_setfenv(L, -2); // [src, udata] +} + +static bool node_check(lua_State *L, TSNode *res) +{ + if (!lua_gettop(L)) { + return 0; + } + if (!lua_isuserdata(L, 1)) { + return 0; + } + // TODO: typecheck! + TSNode *ud = lua_touserdata(L, 1); + *res = *ud; + return true; +} + + +static int node_tostring(lua_State *L) +{ + TSNode node; + if (!node_check(L, &node)) { + return 0; + } + lua_pushstring(L, ""); + lua_concat(L, 3); + return 1; +} + +static int node_range(lua_State *L) +{ + TSNode node; + if (!node_check(L, &node)) { + return 0; + } + TSPoint start = ts_node_start_point(node); + TSPoint end = ts_node_end_point(node); + lua_pushnumber(L, start.row); + lua_pushnumber(L, start.column); + lua_pushnumber(L, end.row); + lua_pushnumber(L, end.column); + return 4; +} + +static int node_start(lua_State *L) +{ + TSNode node; + if (!node_check(L, &node)) { + return 0; + } + TSPoint start = ts_node_start_point(node); + uint32_t start_byte = ts_node_start_byte(node); + lua_pushnumber(L, start.row); + lua_pushnumber(L, start.column); + lua_pushnumber(L, start_byte); + return 3; +} + +static int node_end_byte(lua_State *L) +{ + TSNode node; + if (!node_check(L, &node)) { + return 0; + } + uint32_t end_byte = ts_node_end_byte(node); + lua_pushnumber(L, end_byte); + return 1; +} + +static int node_child_count(lua_State *L) +{ + TSNode node; + if (!node_check(L, &node)) { + return 0; + } + uint32_t count = ts_node_child_count(node); + lua_pushnumber(L, count); + return 1; +} + +static int node_type(lua_State *L) +{ + TSNode node; + if (!node_check(L, &node)) { + return 0; + } + lua_pushstring(L, ts_node_type(node)); + return 1; +} + +static int node_symbol(lua_State *L) +{ + TSNode node; + if (!node_check(L, &node)) { + return 0; + } + TSSymbol symbol = ts_node_symbol(node); + lua_pushnumber(L, symbol); + return 1; +} + +static int node_child(lua_State *L) +{ + TSNode node; + if (!node_check(L, &node)) { + return 0; + } + long num = lua_tointeger(L, 2); + TSNode child = ts_node_child(node, (uint32_t)num); + push_node(L, child); + return 1; +} + +static int node_descendant_for_point_range(lua_State *L) +{ + TSNode node; + if (!node_check(L, &node)) { + return 0; + } + TSPoint start = {(uint32_t)lua_tointeger(L, 2), + (uint32_t)lua_tointeger(L, 3)}; + TSPoint end = {(uint32_t)lua_tointeger(L, 4), + (uint32_t)lua_tointeger(L, 5)}; + TSNode child = ts_node_descendant_for_point_range(node, start, end); + push_node(L, child); + return 1; +} + +static int node_parent(lua_State *L) +{ + TSNode node; + if (!node_check(L, &node)) { + return 0; + } + TSNode parent = ts_node_parent(node); + push_node(L, parent); + return 1; +} + +static int node_to_cursor(lua_State *L) +{ + TSNode node; + if (!node_check(L, &node)) { + return 0; + } + Tslua_propertysheet *sheet = NULL; + + if (lua_gettop(L) >= 2) { + if (!lua_isuserdata(L, 2)) { + return 0; + } + // TODO: typecheck! + sheet = lua_touserdata(L, 2); + } + + + if (ts_node_is_null(node)) { + lua_pushnil(L); // [src, nil] + return 1; + } + Tslua_cursor *c = lua_newuserdata(L, sizeof(Tslua_cursor)); // [src, udata] + c->cursor = ts_tree_cursor_new(node); + c->sheet = sheet; // TODO: GC ref for sheet! + if (c->sheet) { + c->state_id[0] = 0; + c->level = 1; + c->child_index[c->level] = 0; + c->state_id[c->level] = cursor_next_state(c); + } + lua_getfield(L, LUA_ENVIRONINDEX, "cursor-meta"); // [src, udata, meta] + lua_setmetatable(L, -2); // [src, udata] + lua_getfenv(L, 1); // [src, udata, reftable] + lua_setfenv(L, -2); // [src, udata] + return 1; +} + + +static int cursor_gc(lua_State *L) +{ + Tslua_cursor *cursor = cursor_check(L); + if (!cursor) { + return 0; + } + + ts_tree_cursor_delete(&cursor->cursor); + return 0; +} + +static Tslua_cursor *cursor_check(lua_State *L) +{ + if (!lua_gettop(L)) { + return NULL; + } + if (!lua_isuserdata(L, 1)) { + return NULL; + } + // TODO: typecheck! + Tslua_cursor *ud = lua_touserdata(L, 1); + return ud; +} + + +static int cursor_tostring(lua_State *L) +{ + Tslua_cursor *c = cursor_check(L); + if (!c) { + return 0; + } + TSNode node = ts_tree_cursor_current_node(&c->cursor); + if (ts_node_is_null(node)) { + lua_pushstring(L, ""); + return 1; + } + lua_pushstring(L, ""); + lua_concat(L, 3); + return 1; +} + +static int cursor_next_state(Tslua_cursor *c) +{ + int state_id = c->state_id[c->level-1]; + int child_index = c->child_index[c->level]; + PropertyState *s = &c->sheet->states[state_id]; + TSNode current = ts_tree_cursor_current_node(&c->cursor); + int kind_id = (int)ts_node_symbol(current); + + int i = s->kind_first_trans[kind_id]; + if (i == -1) { + return s->default_next_state_id; + } + + for (; i < (int)kv_size(s->kind_trans); i++) { + KindTransition *t = &kv_A(s->kind_trans, i); + if (t->kind_id != kind_id) { + break; + } + + //if (t->regex_id) { + //} + + if (t->child_index >= 0 && t->child_index != child_index) { + continue; + } + return t->next_state_id; + } + + return s->default_next_state_id; +} + +static bool cursor_goto_first_child(Tslua_cursor *c) +{ + if (!ts_tree_cursor_goto_first_child(&c->cursor)) { + return false; + } + if (c->sheet) { + c->level++; + c->child_index[c->level] = 0; + c->state_id[c->level] = cursor_next_state(c); + } + return true; +} + +static bool cursor_goto_next_sibling(Tslua_cursor *c) +{ + if (!ts_tree_cursor_goto_next_sibling(&c->cursor)) { + return false; + } + if (c->sheet) { + c->child_index[c->level]++; + c->state_id[c->level] = cursor_next_state(c); + } + return true; +} + +static bool cursor_goto_parent(Tslua_cursor *c) +{ + if (!ts_tree_cursor_goto_parent(&c->cursor)) { + return false; + } + if (c->sheet) { + c->level--; + } + return true; +} + +static int cursor_forward(lua_State *L) +{ + Tslua_cursor *c = cursor_check(L); + if (!c) { + return 0; + } + TSTreeCursor *cursor = &c->cursor; + + bool status = false; + + int narg = lua_gettop(L); + uint32_t byte_index = 0; + if (narg >= 1) { + byte_index = (uint32_t)lua_tointeger(L, 2); + } + + if (c->sheet && c->level >= 31) { + lua_pushstring(L,"DEPTH EXCEEDED"); + return lua_error(L); + } + + // TODO: use this and use child index from cursor + //status = ts_tree_cursor_goto_first_child_for_byte(cursor, byte_index) != -1; + status = cursor_goto_first_child(c); + + if (status) { + if (byte_index > 0) { + while (true) { + TSNode node = ts_tree_cursor_current_node(cursor); + if (ts_node_end_byte(node) >= byte_index) { + break; + } + // TODO: for a compound node like statement-list, where highlighting + // of each element doesn't depend on previous siblings, this is inefficient + // internal ts_tree_cursor_goto_first_child_for_byte uses binary search. + // we could check what states doesn't have child_index rules. + if(!cursor_goto_next_sibling(c)) { + // if the parent node was in range, we expect some child node to be + lua_pushstring(L, "UNEXPECTED STATE"); + return lua_error(L); + } + } + + } + + goto ret; + } + + while (true) { + status = cursor_goto_next_sibling(c); + if (status) { + break; + } + + // Current node was last a child, look for sibling on higher + // level + status = cursor_goto_parent(c); + if (!status) { // past end of root node + break; + } + } + +ret: + if (status) { + push_node(L, ts_tree_cursor_current_node(cursor)); + if (c->sheet) { + lua_pushnumber(L, c->sheet->states[c->state_id[c->level]].property_id); + return 2; + } else { + return 1; + } + } else { + return 0; + } +} + +static int cursor_debug(lua_State *L) +{ + Tslua_cursor *c = cursor_check(L); + if (!c) { + return 0; + } + + lua_createtable(L, 0, 0); + for (int i = 0; i <= c->level; i++) { + lua_pushinteger(L, c->state_id[i]); + lua_rawseti(L, -2, i); + } + return 1; +} + +// Propertysheet functions +void tslua_push_propertysheet(lua_State *L, int n_states, int n_kinds) +{ + Tslua_propertysheet *sheet = lua_newuserdata(L, sizeof(Tslua_propertysheet)); + sheet->n_states = n_states; + sheet->n_kinds = n_kinds; + sheet->states = xcalloc(n_states, sizeof(*sheet->states)); + for (int i = 0; i < n_states; i++) { + PropertyState *s = &sheet->states[i]; + s->kind_first_trans = xmalloc(n_kinds * sizeof(*s->kind_first_trans)); + memset(s->kind_first_trans, -1, + sheet->n_kinds * sizeof(*s->kind_first_trans)); + kv_init(s->kind_trans); + } + + lua_getfield(L, LUA_REGISTRYINDEX, REG_KEY); // [udata, env] + lua_getfield(L, -1, "propertysheet-meta"); // [udata, env, meta] + lua_setmetatable(L, -3); // [udata, env] + lua_pop(L, 1); // [udata] +} + +static Tslua_propertysheet *propertysheet_check(lua_State *L) +{ + if (!lua_gettop(L)) { + return 0; + } + if (!lua_isuserdata(L, 1)) { + return 0; + } + // TODO: typecheck! + return lua_touserdata(L, 1); +} + +static int propertysheet_gc(lua_State *L) +{ + Tslua_propertysheet *sheet = propertysheet_check(L); + if (!sheet) { + return 0; + } + + // TODO + return 0; +} + +static int propertysheet_tostring(lua_State *L) +{ + Tslua_propertysheet *c = propertysheet_check(L); + if (!c) { + return 0; + } + + lua_pushstring(L, ""); + return 1; +} + +static int propertysheet_add_state(lua_State *L) +{ + Tslua_propertysheet *sheet = propertysheet_check(L); + if (!sheet) { + return 0; + } + + int state_id = lua_tointeger(L, 2); + int default_next_id = lua_tointeger(L, 3); + int property_id = lua_tointeger(L, 4); + + if (state_id >= sheet->n_states) { + lua_pushstring(L, "out of bounds"); + return lua_error(L); + } + + PropertyState *state = &sheet->states[state_id]; + + state->default_next_state_id = default_next_id; + state->property_id = property_id; + + return 0; +} + +static int propertysheet_add_transition(lua_State *L) +{ + Tslua_propertysheet *sheet = propertysheet_check(L); + if (!sheet) { + return 0; + } + + int state_id = lua_tointeger(L, 2); + int kind_id = lua_tointeger(L, 3); + int next_state_id = lua_tointeger(L, 4); + int child_index = lua_isnil(L, 5) ? -1 : lua_tointeger(L, 5); + + if (state_id >= sheet->n_states || kind_id >= sheet->n_kinds) { + lua_pushstring(L, "out of bounds!!"); + return lua_error(L); + } + + PropertyState *state = &sheet->states[state_id]; + if (state->kind_first_trans[kind_id] == -1) { + state->kind_first_trans[kind_id] = kv_size(state->kind_trans); + } else { + if (kv_Z(state->kind_trans, 0).kind_id != kind_id) { + lua_pushstring(L, "disorder!!"); + return lua_error(L); + } + } + + kv_push(state->kind_trans, ((KindTransition){ + .kind_id = kind_id, + .next_state_id = next_state_id, + .child_index = child_index, + .regex_index = -1, + })); + + return 0; +} + diff --git a/src/nvim/lua/tree_sitter.h b/src/nvim/lua/tree_sitter.h new file mode 100644 index 00000000000000..13537726e288f8 --- /dev/null +++ b/src/nvim/lua/tree_sitter.h @@ -0,0 +1,10 @@ +#ifndef NVIM_LUA_TREE_SITTER_H +#define NVIM_LUA_TREE_SITTER_H + +#include + +#ifdef INCLUDE_GENERATED_DECLARATIONS +# include "lua/tree_sitter.h.generated.h" +#endif + +#endif // NVIM_LUA_TREE_SITTER_H diff --git a/src/nvim/screen.c b/src/nvim/screen.c index 84c3f169efb7bb..165597450ecb2c 100644 --- a/src/nvim/screen.c +++ b/src/nvim/screen.c @@ -116,6 +116,7 @@ #include "nvim/window.h" #include "nvim/os/time.h" #include "nvim/api/private/helpers.h" +#include "nvim/api/vim.h" #define MB_FILLER_CHAR '<' /* character used when a double-width character * doesn't fit. */ @@ -2192,6 +2193,8 @@ win_line ( row = startrow; + char *luatext = NULL; + if (!number_only) { // To speed up the loop below, set extra_check when there is linebreak, // trailing white space and/or syntax processing to be done. @@ -2417,6 +2420,34 @@ win_line ( line = ml_get_buf(wp->w_buffer, lnum, FALSE); ptr = line; + if (wp->w_buffer->b_luahl) { + size_t size = STRLEN(line); + if (lua_attr_bufsize < size) { + xfree(lua_attr_buf); + lua_attr_buf = xcalloc(size, sizeof(*lua_attr_buf)); + lua_attr_bufsize = size; + } else { + memset(lua_attr_buf, 0, size * sizeof(*lua_attr_buf)); + } + Error err = ERROR_INIT; + Array args = ARRAY_DICT_INIT; + ADD(args, WINDOW_OBJ(wp->handle)); + ADD(args, BUFFER_OBJ(wp->w_buffer->handle)); + ADD(args, INTEGER_OBJ(lnum-1)); + String s = cstr_to_string(wp->w_buffer->b_luahl); + lua_attr_active = true; + Object o = nvim_execute_lua(s, args, &err); + lua_attr_active = false; + if (o.type == kObjectTypeString) { + luatext = o.data.string.data; + do_virttext = true; + } else if (ERROR_SET(&err)) { + luatext = err.msg; + do_virttext = true; + } + } + + if (has_spell && !number_only) { // For checking first word with a capital skip white space. if (cap_col == 0) { @@ -3383,6 +3414,10 @@ win_line ( } } + if (wp->w_buffer->b_luahl && v > 0 && v < (long)lua_attr_bufsize+1) { + char_attr = hl_combine_attr(char_attr, lua_attr_buf[v-1]); + } + if (wp->w_buffer->terminal) { char_attr = hl_combine_attr(term_attrs[vcol], char_attr); } @@ -3875,8 +3910,14 @@ win_line ( int rightmost_vcol = 0; int i; - VirtText virt_text = do_virttext ? bufhl_info.line->virt_text - : (VirtText)KV_INITIAL_VALUE; + VirtText virt_text; + if (luatext) { + virt_text = (VirtText)KV_INITIAL_VALUE; + kv_push(virt_text, ((VirtTextChunk){.text = luatext, .hl_id = 0})); + } else { + virt_text = do_virttext ? bufhl_info.line->virt_text + : (VirtText)KV_INITIAL_VALUE; + } size_t virt_pos = 0; LineState s = LINE_STATE((char_u *)""); int virt_attr = 0; @@ -4275,6 +4316,7 @@ win_line ( } xfree(p_extra_free); + xfree(luatext); return row; } diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt index db4dae4b3986fd..baa00c8ad15692 100644 --- a/third-party/CMakeLists.txt +++ b/third-party/CMakeLists.txt @@ -41,6 +41,7 @@ option(USE_BUNDLED_LUV "Use the bundled version of luv." ${USE_BUNDLED}) #XXX(tarruda): Lua is only used for debugging the functional test client, no # build it unless explicitly requested option(USE_BUNDLED_LUA "Use the bundled version of lua." OFF) +option(USE_BUNDLED_TREESITTER "Use the bundled version of treesitter." ${USE_BUNDLED}) if(USE_BUNDLED AND MSVC) option(USE_BUNDLED_GETTEXT "Use the bundled version of gettext." ON) @@ -181,6 +182,21 @@ set(GETTEXT_SHA256 ff942af0e438ced4a8b0ea4b0b6e0d6d657157c5e2364de57baa279c1c125 set(LIBICONV_URL https://ftp.gnu.org/pub/gnu/libiconv/libiconv-1.15.tar.gz) set(LIBICONV_SHA256 ccf536620a45458d26ba83887a983b96827001e92a13847b45e4925cc8913178) + +#set(TREESITTER_URL https://github.com/tree-sitter/tree-sitter/archive/f30485f.tar.gz) +#set(TREESITTER_SHA256 cfe00c0b6f423adb082d6a5747d62eef96aab3d4aa9bd3f694524f0639ab272b) +set(TREESITTER_URL https://github.com/tree-sitter/tree-sitter/archive/d51cd86.tar.gz) +set(TREESITTER_SHA256 409a4b126dbc0ed38cb0d7f9bcab4dd5aec6bd7848a150687b12eb0c00da13ce) + +set(TREESITTER_C_URL https://github.com/tree-sitter/tree-sitter-c/archive/5b250e1.tar.gz) +set(TREESITTER_C_SHA256 4651ee825c49c224b28c6da89ec7172ef10f257d65cf3f904d848ac8831ca0f6) + +set(TREESITTER_JAVASCRIPT_URL https://github.com/tree-sitter/tree-sitter-javascript/archive/847ef20.tar.gz) +set(TREESITTER_JAVASCRIPT_SHA256 6f29842524b7616a2d6095b7a0fba47ec8a81978a80578f043e2e1f942b3e05a) + +set(UTF8PROC_URL https://github.com/JuliaStrings/utf8proc/archive/v2.2.0.tar.gz) +set(UTF8PROC_SHA256 3f8fd1dbdb057ee5ba584a539d5cd1b3952141c0338557cb0bdf8cb9cfed5dbf) + if(USE_BUNDLED_UNIBILIUM) include(BuildUnibilium) endif() @@ -229,6 +245,11 @@ if(USE_BUNDLED_LIBICONV) include(BuildLibiconv) endif() +if(USE_BUNDLED_TREESITTER) + include(BuildTreesitter) + include(BuildUtf8proc) +endif() + include(GetBinaryDeps) if(WIN32) diff --git a/third-party/cmake/BuildTreesitter.cmake b/third-party/cmake/BuildTreesitter.cmake new file mode 100644 index 00000000000000..6d465e5807bbf7 --- /dev/null +++ b/third-party/cmake/BuildTreesitter.cmake @@ -0,0 +1,50 @@ +ExternalProject_Add(treesitter +PREFIX ${DEPS_BUILD_DIR} +URL ${TREESITTER_URL} +DOWNLOAD_DIR ${DEPS_DOWNLOAD_DIR}/treesitter +DOWNLOAD_COMMAND ${CMAKE_COMMAND} + -DPREFIX=${DEPS_BUILD_DIR} + -DDOWNLOAD_DIR=${DEPS_DOWNLOAD_DIR}/treesitter + -DURL=${TREESITTER_URL} + -DEXPECTED_SHA256=${TREESITTER_SHA256} + -DTARGET=treesitter + -DUSE_EXISTING_SRC_DIR=${USE_EXISTING_SRC_DIR} + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/DownloadAndExtractFile.cmake +CONFIGURE_COMMAND true +BUILD_COMMAND true +INSTALL_COMMAND true +) + +ExternalProject_Add(treesitter-c +PREFIX ${DEPS_BUILD_DIR} +URL ${TREESITTER_C_URL} +DOWNLOAD_DIR ${DEPS_DOWNLOAD_DIR}/treesitter-c +DOWNLOAD_COMMAND ${CMAKE_COMMAND} + -DPREFIX=${DEPS_BUILD_DIR} + -DDOWNLOAD_DIR=${DEPS_DOWNLOAD_DIR}/treesitter-c + -DURL=${TREESITTER_C_URL} + -DEXPECTED_SHA256=${TREESITTER_C_SHA256} + -DTARGET=treesitter-c + -DUSE_EXISTING_SRC_DIR=${USE_EXISTING_SRC_DIR} + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/DownloadAndExtractFile.cmake +CONFIGURE_COMMAND true +BUILD_COMMAND true +INSTALL_COMMAND true +) + +ExternalProject_Add(treesitter-javascript +PREFIX ${DEPS_BUILD_DIR} +URL ${TREESITTER_JAVASCRIPT_URL} +DOWNLOAD_DIR ${DEPS_DOWNLOAD_DIR}/treesitter-javascript +DOWNLOAD_COMMAND ${CMAKE_COMMAND} + -DPREFIX=${DEPS_BUILD_DIR} + -DDOWNLOAD_DIR=${DEPS_DOWNLOAD_DIR}/treesitter-javascript + -DURL=${TREESITTER_JAVASCRIPT_URL} + -DEXPECTED_SHA256=${TREESITTER_JAVASCRIPT_SHA256} + -DTARGET=treesitter-javascript + -DUSE_EXISTING_SRC_DIR=${USE_EXISTING_SRC_DIR} + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/DownloadAndExtractFile.cmake +CONFIGURE_COMMAND true +BUILD_COMMAND true +INSTALL_COMMAND true +) diff --git a/third-party/cmake/BuildUtf8proc.cmake b/third-party/cmake/BuildUtf8proc.cmake new file mode 100644 index 00000000000000..df287ea459f57c --- /dev/null +++ b/third-party/cmake/BuildUtf8proc.cmake @@ -0,0 +1,16 @@ +ExternalProject_Add(utf8proc +PREFIX ${DEPS_BUILD_DIR} +URL ${UTF8PROC_URL} +DOWNLOAD_DIR ${DEPS_DOWNLOAD_DIR}/utf8proc +DOWNLOAD_COMMAND ${CMAKE_COMMAND} + -DPREFIX=${DEPS_BUILD_DIR} + -DDOWNLOAD_DIR=${DEPS_DOWNLOAD_DIR}/utf8proc + -DURL=${UTF8PROC_URL} + -DEXPECTED_SHA256=${UTF8PROC_SHA256} + -DTARGET=utf8proc + -DUSE_EXISTING_SRC_DIR=${USE_EXISTING_SRC_DIR} + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/DownloadAndExtractFile.cmake +CONFIGURE_COMMAND true +BUILD_COMMAND true +INSTALL_COMMAND true +)