Skip to content

Commit

Permalink
Combine sweeping and moving
Browse files Browse the repository at this point in the history 
This commit combines the sweep step with moving objects.  With this
commit, we can do:

```ruby
GC.start(compact: true)
```

This code will do the following 3 steps:

1. Fully mark the heap
2. Sweep + Move objects
3. Update references

By default, this will compact in order that heap pages are allocated.
In other words, objects will be packed towards older heap pages (as
opposed to heap pages with more pinned objects like `GC.compact` does).
  • Loading branch information
@tenderlove
tenderlove committed May 29, 2020
1 parent c1f6552 commit 02b216e
Show file tree
Hide file tree
Showing 2 changed files with 140 additions and 15 deletions.
147 changes: 136 additions & 11 deletions gc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -480,6 +480,7 @@ typedef enum {
GPR_FLAG_HAVE_FINALIZE = 0x4000,
GPR_FLAG_IMMEDIATE_MARK = 0x8000,
GPR_FLAG_FULL_MARK = 0x10000,
GPR_FLAG_COMPACT = 0x20000,

GPR_DEFAULT_REASON =
(GPR_FLAG_FULL_MARK | GPR_FLAG_IMMEDIATE_MARK |
Expand Down Expand Up @@ -634,6 +635,8 @@ typedef struct rb_heap_struct {
struct heap_page *using_page;
struct list_head pages;
struct heap_page *sweeping_page; /* iterator for .pages */
struct heap_page *compact_cursor;
VALUE moved_list;
#if GC_ENABLE_INCREMENTAL_MARK
struct heap_page *pooled_pages;
#endif
Expand Down Expand Up @@ -667,6 +670,7 @@ typedef struct rb_objspace {
unsigned int gc_stressful: 1;
unsigned int has_hook: 1;
unsigned int during_minor_gc : 1;
unsigned int compact : 1;
#if GC_ENABLE_INCREMENTAL_MARK
unsigned int during_incremental_marking : 1;
#endif
Expand Down Expand Up @@ -4178,17 +4182,71 @@ gc_setup_mark_bits(struct heap_page *page)
memcpy(&page->mark_bits[0], &page->uncollectible_bits[0], HEAP_PAGE_BITMAP_SIZE);
}

static int gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj);
static VALUE gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, VALUE moved_list);

static short
try_move(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *sweep_page, VALUE vp)
{
struct heap_page * cursor = heap->compact_cursor;

while(1) {
bits_t *mark_bits = cursor->mark_bits;
RVALUE * p = cursor->start;
RVALUE * offset = p - NUM_IN_PAGE(p);

/* Find an object to move and move it. Movable objects must be
* marked, so we iterate using the marking bitmap */
for (size_t i = 0; i < HEAP_PAGE_BITMAP_LIMIT; i++) {
bits_t bits = mark_bits[i];

if (bits) {
p = offset + i * BITS_BITLENGTH;

do {
if (bits & 1) {
if (gc_is_moveable_obj(objspace, (VALUE)p)) {
heap->moved_list = gc_move(objspace, (VALUE)p, vp, heap->moved_list);
return 1;
}
}
p++;
bits >>= 1;
} while (bits);
}
}

struct heap_page * next;

next = list_prev(&heap->pages, cursor, page_node);

// Cursors have met, lets quit
if (next == sweep_page) {
break;
} else {
heap->compact_cursor = next;
cursor = next;
}
}

return 0;
}

static inline int
gc_page_sweep(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *sweep_page)
{
int i;
int empty_slots = 0, freed_slots = 0, final_slots = 0;
int empty_slots = 0, freed_slots = 0, final_slots = 0, moved_slots = 0;
RVALUE *p, *pend,*offset;
bits_t *bits, bitset;

gc_report(2, objspace, "page_sweep: start.\n");

sweep_page->flags.before_sweep = FALSE;
if (heap->compact_cursor && sweep_page == heap->compact_cursor) {
/* The compaction cursor and sweep page met, so we need to quit compacting */
heap->compact_cursor = NULL;
}

p = sweep_page->start; pend = p + sweep_page->total_slots;
offset = p - NUM_IN_PAGE(p);
Expand Down Expand Up @@ -4220,20 +4278,51 @@ gc_page_sweep(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *sweep_
}
else {
(void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, sizeof(RVALUE));
heap_page_add_freeobj(objspace, sweep_page, vp);

if (heap->compact_cursor) {
/* If try_move couldn't compact anything, it means
* the cursors have met and there are no objects left that
* /can/ be compacted, so we need to quit. */
if (try_move(objspace, heap, sweep_page, vp)) {
moved_slots++;
} else {
heap->compact_cursor = NULL;
heap_page_add_freeobj(objspace, sweep_page, vp);
}
} else {
heap_page_add_freeobj(objspace, sweep_page, vp);
}

gc_report(3, objspace, "page_sweep: %s is added to freelist\n", obj_info(vp));
freed_slots++;
asan_poison_object(vp);
freed_slots++;
}
break;
}

/* minor cases */
case T_MOVED:
if (!objspace->flags.during_compacting) {
/* When compaction is combined with sweeping, some of the swept pages
* will have T_MOVED objects on them. These need to be kept alive
* until references are finished updating. Once references are finished
* updating, `gc_unlink_moved_list` will clear the T_MOVED slots
* and release them back to the heap. If there are T_MOVED slots
* in the heap and we're _not_ compacting, then it's a bug. */
rb_bug("T_MOVED shouldn't be on the heap unless compacting\n");
}
break;
case T_ZOMBIE:
/* already counted */
break;
case T_NONE:
empty_slots++; /* already freed */
if (heap->compact_cursor) {
if (try_move(objspace, heap, sweep_page, vp)) {
moved_slots++;
} else {
heap->compact_cursor = NULL;
}
}
empty_slots++; /* already freed */
break;
}
}
Expand All @@ -4257,7 +4346,7 @@ gc_page_sweep(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *sweep_
(int)sweep_page->total_slots,
freed_slots, empty_slots, final_slots);

sweep_page->free_slots = freed_slots + empty_slots;
sweep_page->free_slots = (freed_slots + empty_slots) - moved_slots;
objspace->profile.total_freed_objects += freed_slots;
heap_pages_final_slots += final_slots;
sweep_page->final_slots += final_slots;
Expand All @@ -4271,7 +4360,7 @@ gc_page_sweep(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *sweep_

gc_report(2, objspace, "page_sweep: end.\n");

return freed_slots + empty_slots;
return (freed_slots + empty_slots) - moved_slots;
}

/* allocate additional minimum page to work */
Expand Down Expand Up @@ -4456,6 +4545,29 @@ gc_sweep_continue(rb_objspace_t *objspace, rb_heap_t *heap)
gc_exit(objspace, "sweep_continue");
}

static void
gc_compact_start(rb_objspace_t *objspace, rb_heap_t *heap)
{
heap->compact_cursor = list_tail(&heap->pages, struct heap_page, page_node);
objspace->profile.compact_count++;
heap->moved_list = Qfalse;
}

static void gc_update_references(rb_objspace_t * objspace);
static void gc_unlink_moved_list(rb_objspace_t *objspace, VALUE moved_list_head);

static void
gc_compact_finish(rb_objspace_t *objspace, rb_heap_t *heap)
{
heap->compact_cursor = NULL;
gc_update_references(objspace);
rb_clear_constant_cache();
gc_unlink_moved_list(objspace, heap->moved_list);
heap->moved_list = Qfalse;
objspace->flags.compact = 0;
objspace->flags.during_compacting = FALSE;
}

static void
gc_sweep(rb_objspace_t *objspace)
{
Expand All @@ -4468,7 +4580,13 @@ gc_sweep(rb_objspace_t *objspace)
gc_prof_sweep_timer_start(objspace);
#endif
gc_sweep_start(objspace);
if (objspace->flags.compact) {
gc_compact_start(objspace, heap_eden);
}
gc_sweep_rest(objspace);
if (objspace->flags.compact) {
gc_compact_finish(objspace, heap_eden);
}
#if !GC_ENABLE_LAZY_SWEEP
gc_prof_sweep_timer_stop(objspace);
#endif
Expand Down Expand Up @@ -7283,6 +7401,8 @@ gc_start(rb_objspace_t *objspace, int reason)

/* reason may be clobbered, later, so keep set immediate_sweep here */
objspace->flags.immediate_sweep = !!((unsigned)reason & GPR_FLAG_IMMEDIATE_SWEEP);
objspace->flags.compact = !!((unsigned)reason & GPR_FLAG_COMPACT);
objspace->flags.during_compacting = TRUE;

if (!heap_allocated_pages) return FALSE; /* heap is not ready */
if (!(reason & GPR_FLAG_METHOD) && !ready_to_gc(objspace)) return TRUE; /* GC is not allowed */
Expand Down Expand Up @@ -7544,17 +7664,22 @@ garbage_collect_with_gvl(rb_objspace_t *objspace, int reason)
}

static VALUE
gc_start_internal(rb_execution_context_t *ec, VALUE self, VALUE full_mark, VALUE immediate_mark, VALUE immediate_sweep)
gc_start_internal(rb_execution_context_t *ec, VALUE self, VALUE full_mark, VALUE immediate_mark, VALUE immediate_sweep, VALUE compact)
{
rb_objspace_t *objspace = &rb_objspace;
int reason = GPR_FLAG_FULL_MARK |
GPR_FLAG_IMMEDIATE_MARK |
GPR_FLAG_IMMEDIATE_SWEEP |
GPR_FLAG_METHOD;

if (!RTEST(full_mark)) reason &= ~GPR_FLAG_FULL_MARK;
if (!RTEST(immediate_mark)) reason &= ~GPR_FLAG_IMMEDIATE_MARK;
if (!RTEST(immediate_sweep)) reason &= ~GPR_FLAG_IMMEDIATE_SWEEP;
/* For now, compact implies full mark / sweep, so ignore other flags */
if (RTEST(compact)) {
reason |= GPR_FLAG_COMPACT;
} else {
if (!RTEST(full_mark)) reason &= ~GPR_FLAG_FULL_MARK;
if (!RTEST(immediate_mark)) reason &= ~GPR_FLAG_IMMEDIATE_MARK;
if (!RTEST(immediate_sweep)) reason &= ~GPR_FLAG_IMMEDIATE_SWEEP;
}

garbage_collect(objspace, reason);
gc_finalize_deferred(objspace);
Expand Down
8 changes: 4 additions & 4 deletions gc.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,12 @@ module GC
# Note: These keyword arguments are implementation and version dependent. They
# are not guaranteed to be future-compatible, and may be ignored if the
# underlying implementation does not support them.
def self.start full_mark: true, immediate_mark: true, immediate_sweep: true
__builtin_gc_start_internal full_mark, immediate_mark, immediate_sweep
def self.start full_mark: true, immediate_mark: true, immediate_sweep: true, compact: false
__builtin_gc_start_internal full_mark, immediate_mark, immediate_sweep, compact
end

def garbage_collect full_mark: true, immediate_mark: true, immediate_sweep: true
__builtin_gc_start_internal full_mark, immediate_mark, immediate_sweep
__builtin_gc_start_internal full_mark, immediate_mark, immediate_sweep, false
end

# call-seq:
Expand Down Expand Up @@ -188,7 +188,7 @@ def self.verify_compaction_references(toward: nil, double_heap: false)

module ObjectSpace
def garbage_collect full_mark: true, immediate_mark: true, immediate_sweep: true
__builtin_gc_start_internal full_mark, immediate_mark, immediate_sweep
__builtin_gc_start_internal full_mark, immediate_mark, immediate_sweep, false
end

module_function :garbage_collect
Expand Down

0 comments on commit 02b216e

Please sign in to comment.