From 270353d3dde34b266235ef961a5174929fa1128d Mon Sep 17 00:00:00 2001 From: lhchavez Date: Sun, 17 Feb 2019 04:58:04 +0000 Subject: [PATCH] Add a debug allocator This change allows debugging all the allocations performed by libgit2. To enable, run the `./script/leak_detector.py --pipe` script, which makes a FIFO listen at `/tmp/git2go_alloc`. Once that is running, set the environment variable `GIT2GO_DEBUG_ALLOCATOR_LOG=/tmp/git2go_alloc` and run the git2go program. Once the program exits, a leak summary will be printed. In order to reduce the amount of noise due to the static allocations performed by libgit2, it is recommended to call `git.Shutdown()` before exiting the program. --- debug_allocator.c | 192 ++++++++++++++++++++++++++++++++++++++++ git.go | 8 ++ git_dynamic.go | 3 +- git_static.go | 2 +- script/leak_detector.py | 128 +++++++++++++++++++++++++++ 5 files changed, 331 insertions(+), 2 deletions(-) create mode 100644 debug_allocator.c create mode 100755 script/leak_detector.py diff --git a/debug_allocator.c b/debug_allocator.c new file mode 100644 index 00000000..50197ded --- /dev/null +++ b/debug_allocator.c @@ -0,0 +1,192 @@ +#include "_cgo_export.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +static git_allocator _go_git_system_allocator; +static git_allocator _go_git_debug_allocator; + +static int __alloc_fd = -1; + +static void log_alloc_event(char type, const void* ptr, size_t len, const char *file, int line) { + void *btaddr[16]; + char buffer[8192]; + char **strings = NULL; + size_t ptr_size = sizeof(buffer), buffer_size = 0; + int written; + + if (type == 'D') { + written = snprintf(buffer + buffer_size, ptr_size, "%c\t%p\n", type, ptr); + if (written < 0 || written >= ptr_size) { + perror("snprintf"); + abort(); + } + ptr_size -= written; + buffer_size += written; + } else { + size_t i; + int btlen = backtrace(btaddr, 16); + strings = backtrace_symbols(btaddr, btlen); + + written = snprintf(buffer + buffer_size, ptr_size, "%c\t%p\t%zu\t%s:%d", type, ptr, len, file, line); + if (written < 0 || written >= ptr_size) { + perror("snprintf"); + abort(); + } + ptr_size -= written; + buffer_size += written; + + for (i = 0; i < btlen; ++i) { + written = snprintf(buffer + buffer_size, ptr_size, "\t%s", strings[i]); + if (written < 0 || written >= ptr_size) { + perror("snprintf"); + abort(); + } + ptr_size -= written; + buffer_size += written; + } + free(strings); + + written = snprintf(buffer + buffer_size, ptr_size, "\n"); + if (written < 0 || written >= ptr_size) { + perror("snprintf"); + abort(); + } + ptr_size -= written; + buffer_size += written; + } + + if (write(__alloc_fd, buffer, buffer_size) != buffer_size) { + perror("write"); + abort(); + } +} + +static void *_go_git_debug_allocator__malloc(size_t len, const char *file, int line) +{ + void *ptr = _go_git_system_allocator.gmalloc(len, file, line); + if (ptr) + log_alloc_event('A', ptr, len, file, line); + return ptr; +} + +static void *_go_git_debug_allocator__calloc(size_t nelem, size_t elsize, const char *file, int line) +{ + void *ptr = _go_git_system_allocator.gcalloc(nelem, elsize, file, line); + if (ptr) + log_alloc_event('A', ptr, nelem * elsize, file, line); + return ptr; +} + +static char *_go_git_debug_allocator__strdup(const char *str, const char *file, int line) +{ + char *ptr = _go_git_system_allocator.gstrdup(str, file, line); + if (ptr) + log_alloc_event('A', ptr, strlen(ptr) + 1, file, line); + return ptr; +} + +static char *_go_git_debug_allocator__strndup(const char *str, size_t n, const char *file, int line) +{ + char *ptr = _go_git_system_allocator.gstrndup(str, n, file, line); + if (ptr) + log_alloc_event('A', ptr, strlen(ptr) + 1, file, line); + return ptr; +} + +static char *_go_git_debug_allocator__substrdup(const char *start, size_t n, const char *file, int line) +{ + char *ptr = _go_git_system_allocator.gsubstrdup(start, n, file, line); + if (ptr) + log_alloc_event('A', ptr, strlen(ptr) + 1, file, line); + return ptr; +} + +static void *_go_git_debug_allocator__realloc(void *ptr, size_t size, const char *file, int line) +{ + void *new_ptr = _go_git_system_allocator.grealloc(ptr, size, file, line); + if (new_ptr != ptr) { + if (ptr) + log_alloc_event('D', ptr, 0, NULL, 0); + if (new_ptr) + log_alloc_event('A', new_ptr, size, file, line); + } else if (new_ptr) { + log_alloc_event('R', new_ptr, size, file, line); + } + return new_ptr; +} + +static void *_go_git_debug_allocator__reallocarray(void *ptr, size_t nelem, size_t elsize, const char *file, int line) +{ + void *new_ptr = _go_git_system_allocator.greallocarray(ptr, nelem, elsize, file, line); + if (new_ptr != ptr) { + if (ptr) + log_alloc_event('D', ptr, 0, NULL, 0); + if (new_ptr) + log_alloc_event('A', new_ptr, nelem * elsize, file, line); + } else if (new_ptr) { + log_alloc_event('R', new_ptr, nelem * elsize, file, line); + } + return new_ptr; +} + +static void *_go_git_debug_allocator__mallocarray(size_t nelem, size_t elsize, const char *file, int line) +{ + void *ptr = _go_git_system_allocator.gmallocarray(nelem, elsize, file, line); + if (ptr) + log_alloc_event('A', ptr, nelem * elsize, file, line); + return ptr; +} + +static void _go_git_debug_allocator__free(void *ptr) +{ + _go_git_system_allocator.gfree(ptr); + if (ptr) + log_alloc_event('D', ptr, 0, NULL, 0); +} + +int _go_git_setup_debug_allocator(const char *log_path) +{ +#if defined(LIBGIT2_STATIC) + int error; + + __alloc_fd = open(log_path, O_CREAT | O_TRUNC | O_CLOEXEC | O_WRONLY, 0644); + if (__alloc_fd == -1) { + perror("open"); + return -1; + } + + error = git_stdalloc_init_allocator(&_go_git_system_allocator); + if (error < 0) + return error; + _go_git_debug_allocator.gmalloc = _go_git_debug_allocator__malloc; + _go_git_debug_allocator.gcalloc = _go_git_debug_allocator__calloc; + _go_git_debug_allocator.gstrdup = _go_git_debug_allocator__strdup; + _go_git_debug_allocator.gstrndup = _go_git_debug_allocator__strndup; + _go_git_debug_allocator.gsubstrdup = _go_git_debug_allocator__substrdup; + _go_git_debug_allocator.grealloc = _go_git_debug_allocator__realloc; + _go_git_debug_allocator.greallocarray = _go_git_debug_allocator__reallocarray; + _go_git_debug_allocator.gmallocarray = _go_git_debug_allocator__mallocarray; + _go_git_debug_allocator.gfree = _go_git_debug_allocator__free; + error = git_libgit2_opts(GIT_OPT_SET_ALLOCATOR, &_go_git_debug_allocator); + if (error < 0) + return error; + + return 0; +#elif defined(LIBGIT2_DYNAMIC) + fprintf(stderr, "debug allocator is only enabled in static builds\n"); + return -1; +#else +#error no LIBGIT2_STATIC or LIBGIT2_DYNAMIC defined! +#endif +} diff --git a/git.go b/git.go index 0459dde3..874143ef 100644 --- a/git.go +++ b/git.go @@ -3,12 +3,14 @@ package git /* #include #include +extern int _go_git_setup_debug_allocator(const char *log_path); */ import "C" import ( "bytes" "encoding/hex" "errors" + "os" "runtime" "strings" "unsafe" @@ -120,6 +122,12 @@ var pointerHandles *HandleList func init() { pointerHandles = NewHandleList() + if val, ok := os.LookupEnv("GIT2GO_DEBUG_ALLOCATOR_LOG"); ok && val != "" { + if _, err := C._go_git_setup_debug_allocator(C.CString(val)); err != nil { + panic(err) + } + } + C.git_libgit2_init() // Due to the multithreaded nature of Go and its interaction with diff --git a/git_dynamic.go b/git_dynamic.go index b31083ed..0a70da34 100644 --- a/git_dynamic.go +++ b/git_dynamic.go @@ -3,8 +3,9 @@ package git /* -#include #cgo pkg-config: libgit2 +#cgo CFLAGS: -DLIBGIT2_DYNAMIC +#include #if LIBGIT2_VER_MAJOR != 1 || LIBGIT2_VER_MINOR != 0 # error "Invalid libgit2 version; this git2go supports libgit2 v1.0" diff --git a/git_static.go b/git_static.go index 46865fb3..aeba1c56 100644 --- a/git_static.go +++ b/git_static.go @@ -6,11 +6,11 @@ package git #cgo windows CFLAGS: -I${SRCDIR}/static-build/install/include/ #cgo windows LDFLAGS: -L${SRCDIR}/static-build/install/lib/ -lgit2 -lwinhttp #cgo !windows pkg-config: --static ${SRCDIR}/static-build/install/lib/pkgconfig/libgit2.pc +#cgo CFLAGS: -DLIBGIT2_STATIC #include #if LIBGIT2_VER_MAJOR != 1 || LIBGIT2_VER_MINOR != 0 # error "Invalid libgit2 version; this git2go supports libgit2 v1.0" #endif - */ import "C" diff --git a/script/leak_detector.py b/script/leak_detector.py new file mode 100755 index 00000000..f081c9d1 --- /dev/null +++ b/script/leak_detector.py @@ -0,0 +1,128 @@ +#!/usr/bin/python3 +"""Tool to assist in debugging git2go memory leaks. + +In order to use, run this program as root and start the git2go binary with the +`GIT2GO_DEBUG_ALLOCATOR_LOG=/tmp/git2go_alloc` environment variable set. For +best results, make sure that the program exits and calls `git.Shutdown()` at +the end to remove most noise. +""" + +import argparse +import dataclasses +import os +from typing import Dict, Iterable, TextIO, Tuple, Sequence + + +@dataclasses.dataclass +class Allocation: + """A single object allocation.""" + + line: str + size: int + ptr: int + backtrace: Sequence[str] + + +def _receive_allocation_messages( + log: TextIO) -> Iterable[Tuple[str, Allocation]]: + for line in log: + tokens = line.split('\t') + message_type, ptr = tokens[:2] + if message_type == 'D': + yield message_type, Allocation(line='', + size=0, + ptr=int(ptr, 16), + backtrace=()) + else: + yield message_type, Allocation(line=tokens[3], + size=int(tokens[2]), + ptr=int(ptr, 16), + backtrace=tuple(tokens[4:])) + + +@dataclasses.dataclass +class LeakSummaryEntry: + """An entry in the leak summary.""" + + allocation_count: int + allocation_size: int + line: str + backtrace: Sequence[str] + + +def _process_leaked_allocations( + live_allocations: Dict[int, Allocation]) -> None: + """Print a summary of leaked allocations.""" + + if not live_allocations: + print('No leaks!') + return + + backtraces: Dict[Sequence[str], LeakSummaryEntry] = {} + for obj in live_allocations.values(): + if obj.backtrace not in backtraces: + backtraces[obj.backtrace] = LeakSummaryEntry( + 0, 0, obj.line, obj.backtrace) + backtraces[obj.backtrace].allocation_count += 1 + backtraces[obj.backtrace].allocation_size += obj.size + print(f'{"Total size":>20} | {"Average size":>20} | ' + f'{"Allocations":>11} | Filename') + print(f'{"":=<20}=+={"":=<20}=+={"":=<11}=+={"":=<64}') + for entry in sorted(backtraces.values(), + key=lambda e: e.allocation_size, + reverse=True): + print(f'{entry.allocation_size:20} | ' + f'{entry.allocation_size//entry.allocation_count:20} | ' + f'{entry.allocation_count:11} | ' + f'{entry.line}') + for frame in entry.backtrace: + print(f'{"":20} | {"":20} | {"":11} | {frame}') + print(f'{"":-<20}-+-{"":-<20}-+-{"":-<11}-+-{"":-<64}') + print() + + +def _handle_log(log: TextIO) -> None: + """Parse the allocation log.""" + + live_allocations: Dict[int, Allocation] = {} + try: + for message_type, allocation in _receive_allocation_messages(log): + if message_type in ('A', 'R'): + live_allocations[allocation.ptr] = allocation + elif message_type == 'D': + del live_allocations[allocation.ptr] + else: + raise Exception(f'Unknown message type "{message_type}"') + except KeyboardInterrupt: + pass + _process_leaked_allocations(live_allocations) + + +def main() -> None: + """Tool to assist in debugging git2go memory leaks.""" + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--pipe', + action='store_true', + help='Create a FIFO at the specified location') + parser.add_argument('log_path', + metavar='PATH', + default='/tmp/git2go_alloc', + nargs='?', + type=str) + args = parser.parse_args() + + if args.pipe: + try: + os.unlink(args.log_path) + except FileNotFoundError: + pass + os.mkfifo(args.log_path) + print('Capturing allocations, press Ctrl+C to stop...') + + with open(args.log_path, 'r') as log: + _handle_log(log) + + +if __name__ == '__main__': + main()