Add a debug allocator

debug_allocator.c

@@ -0,0 +1,192 @@
+#include "_cgo_export.h"
+
+#include <execinfo.h>
+#include <fcntl.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <git2.h>
+#include <git2/common.h>
+#include <git2/sys/alloc.h>
+
+static git_allocator _go_git_system_allocator;
+static git_allocator _go_git_debug_allocator;
+
+static int __alloc_fd = -1;
+
+static void log_alloc_event(char type, const void* ptr, size_t len, const char *file, int line) {
+	void *btaddr[16];
+	char buffer[8192];
+	char **strings = NULL;
+	size_t ptr_size = sizeof(buffer), buffer_size = 0;
+	int written;
+
+	if (type == 'D') {
+		written = snprintf(buffer + buffer_size, ptr_size, "%c\t%p\n", type, ptr);
+		if (written < 0 || written >= ptr_size) {
+			perror("snprintf");
+			abort();
+		}
+		ptr_size -= written;
+		buffer_size += written;
+	} else {
+		size_t i;
+		int btlen = backtrace(btaddr, 16);
+		strings = backtrace_symbols(btaddr, btlen);
+
+		written = snprintf(buffer + buffer_size, ptr_size, "%c\t%p\t%zu\t%s:%d", type, ptr, len, file, line);
+		if (written < 0 || written >= ptr_size) {
+			perror("snprintf");
+			abort();
+		}
+		ptr_size -= written;
+		buffer_size += written;
+
+		for (i = 0; i < btlen; ++i) {
+			written = snprintf(buffer + buffer_size, ptr_size, "\t%s", strings[i]);
+			if (written < 0 || written >= ptr_size) {
+				perror("snprintf");
+				abort();
+			}
+			ptr_size -= written;
+			buffer_size += written;
+		}
+		free(strings);
+
+		written = snprintf(buffer + buffer_size, ptr_size, "\n");
+		if (written < 0 || written >= ptr_size) {
+			perror("snprintf");
+			abort();
+		}
+		ptr_size -= written;
+		buffer_size += written;
+	}
+
+	if (write(__alloc_fd, buffer, buffer_size) != buffer_size) {
+		perror("write");
+		abort();
+	}
+}
+
+static void *_go_git_debug_allocator__malloc(size_t len, const char *file, int line)
+{
+	void *ptr = _go_git_system_allocator.gmalloc(len, file, line);
+	if (ptr)
+		log_alloc_event('A', ptr, len, file, line);
+	return ptr;
+}
+
+static void *_go_git_debug_allocator__calloc(size_t nelem, size_t elsize, const char *file, int line)
+{
+	void *ptr = _go_git_system_allocator.gcalloc(nelem, elsize, file, line);
+	if (ptr)
+		log_alloc_event('A', ptr, nelem * elsize, file, line);
+	return ptr;
+}
+
+static char *_go_git_debug_allocator__strdup(const char *str, const char *file, int line)
+{
+	char *ptr = _go_git_system_allocator.gstrdup(str, file, line);
+	if (ptr)
+		log_alloc_event('A', ptr, strlen(ptr) + 1, file, line);
+	return ptr;
+}
+
+static char *_go_git_debug_allocator__strndup(const char *str, size_t n, const char *file, int line)
+{
+	char *ptr = _go_git_system_allocator.gstrndup(str, n, file, line);
+	if (ptr)
+		log_alloc_event('A', ptr, strlen(ptr) + 1, file, line);
+	return ptr;
+}
+
+static char *_go_git_debug_allocator__substrdup(const char *start, size_t n, const char *file, int line)
+{
+	char *ptr = _go_git_system_allocator.gsubstrdup(start, n, file, line);
+	if (ptr)
+		log_alloc_event('A', ptr, strlen(ptr) + 1, file, line);
+	return ptr;
+}
+
+static void *_go_git_debug_allocator__realloc(void *ptr, size_t size, const char *file, int line)
+{
+	void *new_ptr = _go_git_system_allocator.grealloc(ptr, size, file, line);
+	if (new_ptr != ptr) {
+		if (ptr)
+			log_alloc_event('D', ptr, 0, NULL, 0);
+		if (new_ptr)
+			log_alloc_event('A', new_ptr, size, file, line);
+	} else if (new_ptr) {
+		log_alloc_event('R', new_ptr, size, file, line);
+	}
+	return new_ptr;
+}
+
+static void *_go_git_debug_allocator__reallocarray(void *ptr, size_t nelem, size_t elsize, const char *file, int line)
+{
+	void *new_ptr = _go_git_system_allocator.greallocarray(ptr, nelem, elsize, file, line);
+	if (new_ptr != ptr) {
+		if (ptr)
+			log_alloc_event('D', ptr, 0, NULL, 0);
+		if (new_ptr)
+			log_alloc_event('A', new_ptr, nelem * elsize, file, line);
+	} else if (new_ptr) {
+		log_alloc_event('R', new_ptr, nelem * elsize, file, line);
+	}
+	return new_ptr;
+}
+
+static void *_go_git_debug_allocator__mallocarray(size_t nelem, size_t elsize, const char *file, int line)
+{
+	void *ptr = _go_git_system_allocator.gmallocarray(nelem, elsize, file, line);
+	if (ptr)
+		log_alloc_event('A', ptr, nelem * elsize, file, line);
+	return ptr;
+}
+
+static void _go_git_debug_allocator__free(void *ptr)
+{
+	_go_git_system_allocator.gfree(ptr);
+	if (ptr)
+		log_alloc_event('D', ptr, 0, NULL, 0);
+}
+
+int _go_git_setup_debug_allocator(const char *log_path)
+{
+#if defined(LIBGIT2_STATIC)
+	int error;
+
+	__alloc_fd = open(log_path, O_CREAT | O_TRUNC | O_CLOEXEC | O_WRONLY, 0644);
+	if (__alloc_fd == -1) {
+		perror("open");
+		return -1;
+	}
+
+	error = git_stdalloc_init_allocator(&_go_git_system_allocator);
+	if (error < 0)
+		return error;
+	_go_git_debug_allocator.gmalloc = _go_git_debug_allocator__malloc;
+	_go_git_debug_allocator.gcalloc = _go_git_debug_allocator__calloc;
+	_go_git_debug_allocator.gstrdup = _go_git_debug_allocator__strdup;
+	_go_git_debug_allocator.gstrndup = _go_git_debug_allocator__strndup;
+	_go_git_debug_allocator.gsubstrdup = _go_git_debug_allocator__substrdup;
+	_go_git_debug_allocator.grealloc = _go_git_debug_allocator__realloc;
+	_go_git_debug_allocator.greallocarray = _go_git_debug_allocator__reallocarray;
+	_go_git_debug_allocator.gmallocarray = _go_git_debug_allocator__mallocarray;
+	_go_git_debug_allocator.gfree = _go_git_debug_allocator__free;
+	error = git_libgit2_opts(GIT_OPT_SET_ALLOCATOR, &_go_git_debug_allocator);
+	if (error < 0)
+		return error;
+
+	return 0;
+#elif defined(LIBGIT2_DYNAMIC)
+	fprintf(stderr, "debug allocator is only enabled in static builds\n");
+	return -1;
+#else
+#error no LIBGIT2_STATIC or LIBGIT2_DYNAMIC defined!
+#endif
+}

git.go

@@ -3,12 +3,14 @@ package git
 /*
 #include <git2.h>
 #include <git2/sys/openssl.h>
+extern int _go_git_setup_debug_allocator(const char *log_path);
 */
 import "C"
 import (
 	"bytes"
 	"encoding/hex"
 	"errors"
+	"os"
 	"runtime"
 	"strings"
 	"unsafe"
@@ -120,6 +122,12 @@ var pointerHandles *HandleList
 func init() {
 	pointerHandles = NewHandleList()
 
+	if val, ok := os.LookupEnv("GIT2GO_DEBUG_ALLOCATOR_LOG"); ok && val != "" {
+		if _, err := C._go_git_setup_debug_allocator(C.CString(val)); err != nil {
+			panic(err)
+		}
+	}
+
 	C.git_libgit2_init()
 
 	// Due to the multithreaded nature of Go and its interaction with

git_dynamic.go

@@ -3,8 +3,9 @@
 package git
 
 /*
-#include <git2.h>
 #cgo pkg-config: libgit2
+#cgo CFLAGS: -DLIBGIT2_DYNAMIC
+#include <git2.h>
 
 #if LIBGIT2_VER_MAJOR != 1 || LIBGIT2_VER_MINOR != 0
 # error "Invalid libgit2 version; this git2go supports libgit2 v1.0"

git_static.go

@@ -6,11 +6,11 @@ package git
 #cgo windows CFLAGS: -I${SRCDIR}/static-build/install/include/
 #cgo windows LDFLAGS: -L${SRCDIR}/static-build/install/lib/ -lgit2 -lwinhttp
 #cgo !windows pkg-config: --static ${SRCDIR}/static-build/install/lib/pkgconfig/libgit2.pc
+#cgo CFLAGS: -DLIBGIT2_STATIC
 #include <git2.h>
 
 #if LIBGIT2_VER_MAJOR != 1 || LIBGIT2_VER_MINOR != 0
 # error "Invalid libgit2 version; this git2go supports libgit2 v1.0"
 #endif
-
 */
 import "C"

script/leak_detector.py

@@ -0,0 +1,128 @@
+#!/usr/bin/python3
+"""Tool to assist in debugging git2go memory leaks.
+
+In order to use, run this program as root and start the git2go binary with the
+`GIT2GO_DEBUG_ALLOCATOR_LOG=/tmp/git2go_alloc` environment variable set. For
+best results, make sure that the program exits and calls `git.Shutdown()` at
+the end to remove most noise.
+"""
+
+import argparse
+import dataclasses
+import os
+from typing import Dict, Iterable, TextIO, Tuple, Sequence
+
+
+@dataclasses.dataclass
+class Allocation:
+    """A single object allocation."""
+
+    line: str
+    size: int
+    ptr: int
+    backtrace: Sequence[str]
+
+
+def _receive_allocation_messages(
+        log: TextIO) -> Iterable[Tuple[str, Allocation]]:
+    for line in log:
+        tokens = line.split('\t')
+        message_type, ptr = tokens[:2]
+        if message_type == 'D':
+            yield message_type, Allocation(line='',
+                                           size=0,
+                                           ptr=int(ptr, 16),
+                                           backtrace=())
+        else:
+            yield message_type, Allocation(line=tokens[3],
+                                           size=int(tokens[2]),
+                                           ptr=int(ptr, 16),
+                                           backtrace=tuple(tokens[4:]))
+
+
+@dataclasses.dataclass
+class LeakSummaryEntry:
+    """An entry in the leak summary."""
+
+    allocation_count: int
+    allocation_size: int
+    line: str
+    backtrace: Sequence[str]
+
+
+def _process_leaked_allocations(
+        live_allocations: Dict[int, Allocation]) -> None:
+    """Print a summary of leaked allocations."""
+
+    if not live_allocations:
+        print('No leaks!')
+        return
+
+    backtraces: Dict[Sequence[str], LeakSummaryEntry] = {}
+    for obj in live_allocations.values():
+        if obj.backtrace not in backtraces:
+            backtraces[obj.backtrace] = LeakSummaryEntry(
+                0, 0, obj.line, obj.backtrace)
+        backtraces[obj.backtrace].allocation_count += 1
+        backtraces[obj.backtrace].allocation_size += obj.size
+    print(f'{"Total size":>20} | {"Average size":>20} | '
+          f'{"Allocations":>11} | Filename')
+    print(f'{"":=<20}=+={"":=<20}=+={"":=<11}=+={"":=<64}')
+    for entry in sorted(backtraces.values(),
+                        key=lambda e: e.allocation_size,
+                        reverse=True):
+        print(f'{entry.allocation_size:20} | '
+              f'{entry.allocation_size//entry.allocation_count:20} | '
+              f'{entry.allocation_count:11} | '
+              f'{entry.line}')
+        for frame in entry.backtrace:
+            print(f'{"":20} | {"":20} | {"":11} | {frame}')
+        print(f'{"":-<20}-+-{"":-<20}-+-{"":-<11}-+-{"":-<64}')
+    print()
+
+
+def _handle_log(log: TextIO) -> None:
+    """Parse the allocation log."""
+
+    live_allocations: Dict[int, Allocation] = {}
+    try:
+        for message_type, allocation in _receive_allocation_messages(log):
+            if message_type in ('A', 'R'):
+                live_allocations[allocation.ptr] = allocation
+            elif message_type == 'D':
+                del live_allocations[allocation.ptr]
+            else:
+                raise Exception(f'Unknown message type "{message_type}"')
+    except KeyboardInterrupt:
+        pass
+    _process_leaked_allocations(live_allocations)
+
+
+def main() -> None:
+    """Tool to assist in debugging git2go memory leaks."""
+
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--pipe',
+                        action='store_true',
+                        help='Create a FIFO at the specified location')
+    parser.add_argument('log_path',
+                        metavar='PATH',
+                        default='/tmp/git2go_alloc',
+                        nargs='?',
+                        type=str)
+    args = parser.parse_args()
+
+    if args.pipe:
+        try:
+            os.unlink(args.log_path)
+        except FileNotFoundError:
+            pass
+        os.mkfifo(args.log_path)
+        print('Capturing allocations, press Ctrl+C to stop...')
+
+    with open(args.log_path, 'r') as log:
+        _handle_log(log)
+
+
+if __name__ == '__main__':
+    main()