feat: expose the allocator and array header files for external scanners

cli/build.rs

@@ -13,6 +13,19 @@ fn main() {
 
     let rust_binding_version = read_rust_binding_version();
     println!("cargo:rustc-env=RUST_BINDING_VERSION={rust_binding_version}");
+
+    #[cfg(any(
+        target_os = "linux",
+        target_os = "android",
+        target_os = "freebsd",
+        target_os = "openbsd",
+        target_os = "netbsd",
+        target_os = "dragonfly",
+    ))]
+    println!("cargo:rustc-link-arg=-Wl,--dynamic-list=cli/dynamic-symbols.txt");
+
+    #[cfg(any(target_os = "macos", target_os = "ios"))]
+    println!("cargo:rustc-link-arg=-Wl,-exported_symbols_list,cli/dynamic-symbols-darwin.txt");
 }
 
 fn web_playground_files_present() -> bool {

cli/dynamic-symbols-darwin.txt

@@ -0,0 +1,4 @@
+_ts_current_malloc
+_ts_current_calloc
+_ts_current_realloc
+_ts_current_free

cli/dynamic-symbols.txt

@@ -0,0 +1,6 @@
+{
+  ts_current_malloc;
+  ts_current_calloc;
+  ts_current_realloc;
+  ts_current_free;
+};

cli/loader/src/lib.rs

@@ -529,6 +529,7 @@ impl Loader {
             .cpp(true)
             .opt_level(2)
             .cargo_metadata(false)
+            .cargo_warnings(false)
             .target(BUILD_TARGET)
             .host(BUILD_TARGET)
             .flag_if_supported("-Werror=implicit-function-declaration");
@@ -584,10 +585,6 @@ impl Loader {
                 command.arg("-O2");
             }
 
-            // For conditional compilation of external scanner code when
-            // used internally by `tree-siteer parse` and other sub commands.
-            command.arg("-DTREE_SITTER_INTERNAL_BUILD");
-
             if let Some(scanner_path) = scanner_path.as_ref() {
                 if scanner_path.extension() == Some("c".as_ref()) {
                     command.arg("-xc").arg("-std=c99").arg(scanner_path);
@@ -599,6 +596,14 @@ impl Loader {
             command.arg("-xc").arg(parser_path);
         }
 
+        // For conditional compilation of external scanner code when
+        // used internally by `tree-sitter parse` and other sub commands.
+        command.arg("-DTREE_SITTER_INTERNAL_BUILD");
+
+        // Always use the same allocator in the CLI as any scanner, useful for debugging and
+        // tracking memory leaks in tests.
+        command.arg("-DTS_REUSE_ALLOCATOR");
+
         let output = command.output().with_context(|| {
             format!("Failed to execute the C compiler with the following command:\n{command:?}")
         })?;

cli/src/generate/mod.rs

@@ -89,6 +89,8 @@ pub fn generate_parser_in_directory(
 
     write_file(&src_path.join("parser.c"), c_code)?;
     write_file(&src_path.join("node-types.json"), node_types_json)?;
+    write_file(&header_path.join("alloc.h"), tree_sitter::ALLOC_HEADER)?;
+    write_file(&header_path.join("array.h"), tree_sitter::ARRAY_HEADER)?;
     write_file(&header_path.join("parser.h"), tree_sitter::PARSER_HEADER)?;
 
     if generate_bindings {

cli/src/generate/templates/alloc.h

@@ -0,0 +1,60 @@
+#ifndef TREE_SITTER_ALLOC_H_
+#define TREE_SITTER_ALLOC_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#ifdef _WIN32
+#define TS_PUBLIC __declspec(dllexport)
+#else
+#define TS_PUBLIC __attribute__((visibility("default")))
+#endif
+
+TS_PUBLIC extern void *(*ts_current_malloc)(size_t);
+TS_PUBLIC extern void *(*ts_current_calloc)(size_t, size_t);
+TS_PUBLIC extern void *(*ts_current_realloc)(void *, size_t);
+TS_PUBLIC extern void (*ts_current_free)(void *);
+
+// Allow clients to override allocation functions
+#ifdef TS_REUSE_ALLOCATOR
+
+#ifndef ts_malloc
+#define ts_malloc  ts_current_malloc
+#endif
+#ifndef ts_calloc
+#define ts_calloc  ts_current_calloc
+#endif
+#ifndef ts_realloc
+#define ts_realloc ts_current_realloc
+#endif
+#ifndef ts_free
+#define ts_free    ts_current_free
+#endif
+
+#else
+
+#ifndef ts_malloc
+#define ts_malloc  malloc
+#endif
+#ifndef ts_calloc
+#define ts_calloc  calloc
+#endif
+#ifndef ts_realloc
+#define ts_realloc realloc
+#endif
+#ifndef ts_free
+#define ts_free    free
+#endif
+
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // TREE_SITTER_ALLOC_H_

cli/src/tests/helpers/fixtures.rs

@@ -107,6 +107,25 @@ pub fn get_test_language(name: &str, parser_code: &str, path: Option<&Path>) ->
 
     let header_path = src_dir.join("tree_sitter");
     fs::create_dir_all(&header_path).unwrap();
+
+    fs::write(header_path.join("alloc.h"), tree_sitter::PARSER_HEADER)
+        .with_context(|| {
+            format!(
+                "Failed to write {:?}",
+                header_path.join("alloc.h").file_name().unwrap()
+            )
+        })
+        .unwrap();
+
+    fs::write(header_path.join("array.h"), tree_sitter::PARSER_HEADER)
+        .with_context(|| {
+            format!(
+                "Failed to write {:?}",
+                header_path.join("array.h").file_name().unwrap()
+            )
+        })
+        .unwrap();
+
     fs::write(header_path.join("parser.h"), tree_sitter::PARSER_HEADER)
         .with_context(|| {
             format!(

lib/binding_rust/lib.rs

@@ -44,6 +44,8 @@ pub const LANGUAGE_VERSION: usize = ffi::TREE_SITTER_LANGUAGE_VERSION as usize;
 pub const MIN_COMPATIBLE_LANGUAGE_VERSION: usize =
     ffi::TREE_SITTER_MIN_COMPATIBLE_LANGUAGE_VERSION as usize;
 
+pub const ALLOC_HEADER: &str = include_str!("../../cli/src/generate/templates/alloc.h");
+pub const ARRAY_HEADER: &str = include_str!("../src/array.h");
 pub const PARSER_HEADER: &str = include_str!("../src/parser.h");
 
 /// An opaque object that defines how to parse a particular language. The code for each

lib/src/alloc.c

@@ -29,10 +29,10 @@ static void *ts_realloc_default(void *buffer, size_t size) {
 }
 
 // Allow clients to override allocation functions dynamically
-void *(*ts_current_malloc)(size_t) = ts_malloc_default;
-void *(*ts_current_calloc)(size_t, size_t) = ts_calloc_default;
-void *(*ts_current_realloc)(void *, size_t) = ts_realloc_default;
-void (*ts_current_free)(void *) = free;
+TS_PUBLIC void *(*ts_current_malloc)(size_t) = ts_malloc_default;
+TS_PUBLIC void *(*ts_current_calloc)(size_t, size_t) = ts_calloc_default;
+TS_PUBLIC void *(*ts_current_realloc)(void *, size_t) = ts_realloc_default;
+TS_PUBLIC void (*ts_current_free)(void *) = free;
 
 void ts_set_allocator(
   void *(*new_malloc)(size_t size),

lib/src/alloc.h

@@ -1,20 +1,24 @@
 #ifndef TREE_SITTER_ALLOC_H_
 #define TREE_SITTER_ALLOC_H_
 
-#include "tree_sitter/api.h"
-
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#include <stdlib.h>
 #include <stdbool.h>
 #include <stdio.h>
+#include <stdlib.h>
+
+#ifdef _WIN32
+#define TS_PUBLIC __declspec(dllexport)
+#else
+#define TS_PUBLIC __attribute__((visibility("default")))
+#endif
 
-extern void *(*ts_current_malloc)(size_t);
-extern void *(*ts_current_calloc)(size_t, size_t);
-extern void *(*ts_current_realloc)(void *, size_t);
-extern void (*ts_current_free)(void *);
+TS_PUBLIC extern void *(*ts_current_malloc)(size_t);
+TS_PUBLIC extern void *(*ts_current_calloc)(size_t, size_t);
+TS_PUBLIC extern void *(*ts_current_realloc)(void *, size_t);
+TS_PUBLIC extern void (*ts_current_free)(void *);
 
 // Allow clients to override allocation functions
 #ifndef ts_malloc
@@ -34,4 +38,4 @@ extern void (*ts_current_free)(void *);
 }
 #endif
 
-#endif  // TREE_SITTER_ALLOC_H_
+#endif // TREE_SITTER_ALLOC_H_