Skip to content

Commit

Permalink
Fix segfault in rs_stack_mark
Browse files Browse the repository at this point in the history
I have no context over that gem, but a crash and gdb session
show us marking an empty stack:

```
(gdb) p *stack
$1 = {capacity = 0, top = 0, contents = 0x0}
```

Since we iterate with `i <= top`, we try to mark a NULL pointer
when the stack is empty.
  • Loading branch information
byroot committed Dec 8, 2023
1 parent c2923d9 commit b6002ee
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 15 deletions.
21 changes: 9 additions & 12 deletions ext/rotoscope/rotoscope.c
Original file line number Diff line number Diff line change
Expand Up @@ -146,24 +146,21 @@ static void rs_dealloc(void *data) {
xfree(config);
}

static size_t rs_memsize(const void *data) {
return sizeof(Rotoscope);
}
static size_t rs_memsize(const void *data) { return sizeof(Rotoscope); }

static const rb_data_type_t rs_data_type = {
.wrap_struct_name = "Rotoscope",
.function = {
.dmark = rs_gc_mark,
.dfree = rs_dealloc,
.dsize = rs_memsize,
},
.flags = RUBY_TYPED_FREE_IMMEDIATELY
};
.function =
{
.dmark = rs_gc_mark,
.dfree = rs_dealloc,
.dsize = rs_memsize,
},
.flags = RUBY_TYPED_FREE_IMMEDIATELY};

static VALUE rs_alloc(VALUE klass) {
Rotoscope *config;
VALUE self =
TypedData_Make_Struct(klass, Rotoscope, &rs_data_type, config);
VALUE self = TypedData_Make_Struct(klass, Rotoscope, &rs_data_type, config);
config->self = self;
config->pid = getpid();
config->tid = current_thread_id();
Expand Down
8 changes: 5 additions & 3 deletions ext/rotoscope/stack.c
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,10 @@ void rs_stack_init(rs_stack_t *stack, unsigned int capacity) {
}

void rs_stack_mark(rs_stack_t *stack) {
for (int i = 0; i <= stack->top; i++) {
rs_stack_frame_t *frame = &stack->contents[i];
rs_method_desc_mark(&frame->method);
if (stack->contents) {
for (int i = 0; i <= stack->top; i++) {
rs_stack_frame_t *frame = &stack->contents[i];
rs_method_desc_mark(&frame->method);
}
}
}

0 comments on commit b6002ee

Please sign in to comment.