From b6002ee161d593db75c6d5f363ca3fd6702e162c Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Fri, 8 Dec 2023 16:32:40 +0100 Subject: [PATCH] Fix segfault in rs_stack_mark I have no context over that gem, but a crash and gdb session show us marking an empty stack: ``` (gdb) p *stack $1 = {capacity = 0, top = 0, contents = 0x0} ``` Since we iterate with `i <= top`, we try to mark a NULL pointer when the stack is empty. --- ext/rotoscope/rotoscope.c | 21 +++++++++------------ ext/rotoscope/stack.c | 8 +++++--- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/ext/rotoscope/rotoscope.c b/ext/rotoscope/rotoscope.c index 91d101e..37dbb93 100644 --- a/ext/rotoscope/rotoscope.c +++ b/ext/rotoscope/rotoscope.c @@ -146,24 +146,21 @@ static void rs_dealloc(void *data) { xfree(config); } -static size_t rs_memsize(const void *data) { - return sizeof(Rotoscope); -} +static size_t rs_memsize(const void *data) { return sizeof(Rotoscope); } static const rb_data_type_t rs_data_type = { .wrap_struct_name = "Rotoscope", - .function = { - .dmark = rs_gc_mark, - .dfree = rs_dealloc, - .dsize = rs_memsize, - }, - .flags = RUBY_TYPED_FREE_IMMEDIATELY -}; + .function = + { + .dmark = rs_gc_mark, + .dfree = rs_dealloc, + .dsize = rs_memsize, + }, + .flags = RUBY_TYPED_FREE_IMMEDIATELY}; static VALUE rs_alloc(VALUE klass) { Rotoscope *config; - VALUE self = - TypedData_Make_Struct(klass, Rotoscope, &rs_data_type, config); + VALUE self = TypedData_Make_Struct(klass, Rotoscope, &rs_data_type, config); config->self = self; config->pid = getpid(); config->tid = current_thread_id(); diff --git a/ext/rotoscope/stack.c b/ext/rotoscope/stack.c index b4dd279..a0fe9a7 100644 --- a/ext/rotoscope/stack.c +++ b/ext/rotoscope/stack.c @@ -62,8 +62,10 @@ void rs_stack_init(rs_stack_t *stack, unsigned int capacity) { } void rs_stack_mark(rs_stack_t *stack) { - for (int i = 0; i <= stack->top; i++) { - rs_stack_frame_t *frame = &stack->contents[i]; - rs_method_desc_mark(&frame->method); + if (stack->contents) { + for (int i = 0; i <= stack->top; i++) { + rs_stack_frame_t *frame = &stack->contents[i]; + rs_method_desc_mark(&frame->method); + } } }