feat!: expose the allocator and array header files for external scanners

cli/build.rs

@@ -13,6 +13,19 @@ fn main() {
 
     let rust_binding_version = read_rust_binding_version();
     println!("cargo:rustc-env=RUST_BINDING_VERSION={rust_binding_version}");
+
+    #[cfg(any(
+        target_os = "linux",
+        target_os = "android",
+        target_os = "freebsd",
+        target_os = "openbsd",
+        target_os = "netbsd",
+        target_os = "dragonfly",
+    ))]
+    println!("cargo:rustc-link-arg=-Wl,--dynamic-list=cli/dynamic-symbols.txt");
+
+    #[cfg(any(target_os = "macos", target_os = "ios"))]
+    println!("cargo:rustc-link-arg=-Wl,-exported_symbols_list,cli/dynamic-symbols-darwin.txt");
 }
 
 fn web_playground_files_present() -> bool {

cli/dynamic-symbols-darwin.txt

@@ -0,0 +1,4 @@
+_ts_current_malloc
+_ts_current_calloc
+_ts_current_realloc
+_ts_current_free

cli/dynamic-symbols.txt

@@ -0,0 +1,6 @@
+{
+  ts_current_malloc;
+  ts_current_calloc;
+  ts_current_realloc;
+  ts_current_free;
+};

cli/loader/src/lib.rs

@@ -529,6 +529,7 @@ impl Loader {
             .cpp(true)
             .opt_level(2)
             .cargo_metadata(false)
+            .cargo_warnings(false)
             .target(BUILD_TARGET)
             .host(BUILD_TARGET)
             .flag_if_supported("-Werror=implicit-function-declaration");

cli/src/generate/mod.rs

@@ -89,6 +89,8 @@ pub fn generate_parser_in_directory(
 
     write_file(&src_path.join("parser.c"), c_code)?;
     write_file(&src_path.join("node-types.json"), node_types_json)?;
+    write_file(&header_path.join("alloc.h"), tree_sitter::ALLOC_HEADER)?;
+    write_file(&header_path.join("array.h"), tree_sitter::ARRAY_HEADER)?;
     write_file(&header_path.join("parser.h"), tree_sitter::PARSER_HEADER)?;
 
     if generate_bindings {

cli/src/tests/helpers/fixtures.rs

@@ -107,6 +107,25 @@ pub fn get_test_language(name: &str, parser_code: &str, path: Option<&Path>) ->
 
     let header_path = src_dir.join("tree_sitter");
     fs::create_dir_all(&header_path).unwrap();
+
+    fs::write(header_path.join("alloc.h"), tree_sitter::PARSER_HEADER)
+        .with_context(|| {
+            format!(
+                "Failed to write {:?}",
+                header_path.join("alloc.h").file_name().unwrap()
+            )
+        })
+        .unwrap();
+
+    fs::write(header_path.join("array.h"), tree_sitter::PARSER_HEADER)
+        .with_context(|| {
+            format!(
+                "Failed to write {:?}",
+                header_path.join("array.h").file_name().unwrap()
+            )
+        })
+        .unwrap();
+
     fs::write(header_path.join("parser.h"), tree_sitter::PARSER_HEADER)
         .with_context(|| {
             format!(

lib/binding_rust/lib.rs

@@ -44,6 +44,8 @@ pub const LANGUAGE_VERSION: usize = ffi::TREE_SITTER_LANGUAGE_VERSION as usize;
 pub const MIN_COMPATIBLE_LANGUAGE_VERSION: usize =
     ffi::TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION as usize;
 
+pub const ALLOC_HEADER: &str = include_str!("../src/alloc.h");
+pub const ARRAY_HEADER: &str = include_str!("../src/array.h");
 pub const PARSER_HEADER: &str = include_str!("../src/parser.h");
 
 /// An opaque object that defines how to parse a particular language. The code for each

lib/src/alloc.c

@@ -29,10 +29,10 @@ static void *ts_realloc_default(void *buffer, size_t size) {
 }
 
 // Allow clients to override allocation functions dynamically
-void *(*ts_current_malloc)(size_t) = ts_malloc_default;
-void *(*ts_current_calloc)(size_t, size_t) = ts_calloc_default;
-void *(*ts_current_realloc)(void *, size_t) = ts_realloc_default;
-void (*ts_current_free)(void *) = free;
+TS_PUBLIC void *(*ts_current_malloc)(size_t) = ts_malloc_default;
+TS_PUBLIC void *(*ts_current_calloc)(size_t, size_t) = ts_calloc_default;
+TS_PUBLIC void *(*ts_current_realloc)(void *, size_t) = ts_realloc_default;
+TS_PUBLIC void (*ts_current_free)(void *) = free;
 
 void ts_set_allocator(
   void *(*new_malloc)(size_t size),

lib/src/alloc.h

@@ -1,20 +1,24 @@
 #ifndef TREE_SITTER_ALLOC_H_
 #define TREE_SITTER_ALLOC_H_
 
-#include "tree_sitter/api.h"
-
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#include <stdlib.h>
 #include <stdbool.h>
 #include <stdio.h>
+#include <stdlib.h>
+
+#ifdef _WIN32
+#define TS_PUBLIC __declspec(dllexport)
+#else
+#define TS_PUBLIC __attribute__((visibility("default")))
+#endif
 
-extern void *(*ts_current_malloc)(size_t);
-extern void *(*ts_current_calloc)(size_t, size_t);
-extern void *(*ts_current_realloc)(void *, size_t);
-extern void (*ts_current_free)(void *);
+TS_PUBLIC extern void *(*ts_current_malloc)(size_t);
+TS_PUBLIC extern void *(*ts_current_calloc)(size_t, size_t);
+TS_PUBLIC extern void *(*ts_current_realloc)(void *, size_t);
+TS_PUBLIC extern void (*ts_current_free)(void *);
 
 // Allow clients to override allocation functions
 #ifndef ts_malloc
@@ -34,4 +38,4 @@ extern void (*ts_current_free)(void *);
 }
 #endif
 
-#endif  // TREE_SITTER_ALLOC_H_
+#endif // TREE_SITTER_ALLOC_H_

lib/src/array.h

@@ -5,12 +5,13 @@
 extern "C" {
 #endif
 
-#include <string.h>
-#include <stdlib.h>
-#include <stdint.h>
+#include "./alloc.h"
+
 #include <assert.h>
 #include <stdbool.h>
-#include "./alloc.h"
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
 
 #define Array(T)       \
   struct {             \
@@ -35,19 +36,19 @@ extern "C" {
 #define array_clear(self) ((self)->size = 0)
 
 #define array_reserve(self, new_capacity) \
-  array__reserve((VoidArray *)(self), array__elem_size(self), new_capacity)
+  array__reserve((Array *)(self), array__elem_size(self), new_capacity)
 
 // Free any memory allocated for this array.
-#define array_delete(self) array__delete((VoidArray *)(self))
+#define array_delete(self) array__delete((Array *)(self))
 
 #define array_push(self, element)                            \
-  (array__grow((VoidArray *)(self), 1, array__elem_size(self)), \
+  (array__grow((Array *)(self), 1, array__elem_size(self)), \
    (self)->contents[(self)->size++] = (element))
 
 // Increase the array's size by a given number of elements, reallocating
 // if necessary. New elements are zero-initialized.
 #define array_grow_by(self, count) \
-  (array__grow((VoidArray *)(self), count, array__elem_size(self)), \
+  (array__grow((Array *)(self), count, array__elem_size(self)), \
    memset((self)->contents + (self)->size, 0, (count) * array__elem_size(self)), \
    (self)->size += (count))
 
@@ -58,7 +59,7 @@ extern "C" {
 // `contents` pointer.
 #define array_extend(self, count, contents)                    \
   array__splice(                                               \
-    (VoidArray *)(self), array__elem_size(self), (self)->size, \
+    (Array *)(self), array__elem_size(self), (self)->size, \
     0, count,  contents                                        \
   )
 
@@ -67,25 +68,25 @@ extern "C" {
 // `new_contents` pointer.
 #define array_splice(self, _index, old_count, new_count, new_contents)  \
   array__splice(                                                       \
-    (VoidArray *)(self), array__elem_size(self), _index,                \
+    (Array *)(self), array__elem_size(self), _index,                \
     old_count, new_count, new_contents                                 \
   )
 
 // Insert one `element` into the array at the given `index`.
 #define array_insert(self, _index, element) \
-  array__splice((VoidArray *)(self), array__elem_size(self), _index, 0, 1, &(element))
+  array__splice((Array *)(self), array__elem_size(self), _index, 0, 1, &(element))
 
 // Remove one `element` from the array at the given `index`.
 #define array_erase(self, _index) \
-  array__erase((VoidArray *)(self), array__elem_size(self), _index)
+  array__erase((Array *)(self), array__elem_size(self), _index)
 
 #define array_pop(self) ((self)->contents[--(self)->size])
 
 #define array_assign(self, other) \
-  array__assign((VoidArray *)(self), (const VoidArray *)(other), array__elem_size(self))
+  array__assign((Array *)(self), (const Array *)(other), array__elem_size(self))
 
 #define array_swap(self, other) \
-  array__swap((VoidArray *)(self), (VoidArray *)(other))
+  array__swap((Array *)(self), (Array *)(other))
 
 // Search a sorted array for a given `needle` value, using the given `compare`
 // callback to determine the order.
@@ -127,11 +128,11 @@ extern "C" {
 
 // Private
 
-typedef Array(void) VoidArray;
+typedef Array(void) Array;
 
 #define array__elem_size(self) sizeof(*(self)->contents)
 
-static inline void array__delete(VoidArray *self) {
+static inline void array__delete(Array *self) {
   if (self->contents) {
     ts_free(self->contents);
     self->contents = NULL;
@@ -140,7 +141,7 @@ static inline void array__delete(VoidArray *self) {
   }
 }
 
-static inline void array__erase(VoidArray *self, size_t element_size,
+static inline void array__erase(Array *self, size_t element_size,
                                 uint32_t index) {
   assert(index < self->size);
   char *contents = (char *)self->contents;
@@ -149,7 +150,7 @@ static inline void array__erase(VoidArray *self, size_t element_size,
   self->size--;
 }
 
-static inline void array__reserve(VoidArray *self, size_t element_size, uint32_t new_capacity) {
+static inline void array__reserve(Array *self, size_t element_size, uint32_t new_capacity) {
   if (new_capacity > self->capacity) {
     if (self->contents) {
       self->contents = ts_realloc(self->contents, new_capacity * element_size);
@@ -160,19 +161,19 @@ static inline void array__reserve(VoidArray *self, size_t element_size, uint32_t
   }
 }
 
-static inline void array__assign(VoidArray *self, const VoidArray *other, size_t element_size) {
+static inline void array__assign(Array *self, const Array *other, size_t element_size) {
   array__reserve(self, element_size, other->size);
   self->size = other->size;
   memcpy(self->contents, other->contents, self->size * element_size);
 }
 
-static inline void array__swap(VoidArray *self, VoidArray *other) {
-  VoidArray swap = *other;
+static inline void array__swap(Array *self, Array *other) {
+  Array swap = *other;
   *other = *self;
   *self = swap;
 }
 
-static inline void array__grow(VoidArray *self, uint32_t count, size_t element_size) {
+static inline void array__grow(Array *self, uint32_t count, size_t element_size) {
   uint32_t new_size = self->size + count;
   if (new_size > self->capacity) {
     uint32_t new_capacity = self->capacity * 2;
@@ -182,7 +183,7 @@ static inline void array__grow(VoidArray *self, uint32_t count, size_t element_s
   }
 }
 
-static inline void array__splice(VoidArray *self, size_t element_size,
+static inline void array__splice(Array *self, size_t element_size,
                                  uint32_t index, uint32_t old_count,
                                  uint32_t new_count, const void *elements) {
   uint32_t new_size = self->size + new_count - old_count;