Permalink
Browse files

introduce TransientHeap. [Bug #14858]

* transient_heap.c, transient_heap.h: implement TransientHeap (theap).
  theap is designed for Ruby's object system. theap is like Eden heap
  on generational GC terminology. theap allocation is very fast because
  it only needs to bump up pointer and deallocation is also fast because
  we don't do anything. However we need to evacuate (Copy GC terminology)
  if theap memory is long-lived. Evacuation logic is needed for each type.

  See [Bug #14858] for details.

* array.c: Now, theap for T_ARRAY is supported.

  ary_heap_alloc() tries to allocate memory area from theap. If this trial
  sccesses, this array has theap ptr and RARRAY_TRANSIENT_FLAG is turned on.
  We don't need to free theap ptr.

* ruby.h: RARRAY_CONST_PTR() returns malloc'ed memory area. It menas that
  if ary is allocated at theap, force evacuation to malloc'ed memory.
  It makes programs slow, but very compatible with current code because
  theap memory can be evacuated (theap memory will be recycled).

  If you want to get transient heap ptr, use RARRAY_CONST_PTR_TRANSIENT()
  instead of RARRAY_CONST_PTR(). If you can't understand when evacuation
  will occur, use RARRAY_CONST_PTR().

(re-commit of r65444)


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@65449 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
  • Loading branch information...
ko1 committed Oct 30, 2018
1 parent 69b8ffc commit 312b105d0e263a36c129a14f2681fe16905c0244
Showing with 1,371 additions and 126 deletions.
  1. +326 −72 array.c
  2. +10 −0 common.mk
  3. +3 −3 compile.c
  4. +5 −0 debug_counter.h
  5. +4 −2 enum.c
  6. +66 −23 gc.c
  7. +25 −5 include/ruby/ruby.h
  8. +1 −0 inits.c
  9. +1 −1 insns.def
  10. +24 −1 internal.h
  11. +2 −2 string.c
  12. +15 −0 test/ruby/test_enum.rb
  13. +834 −0 transient_heap.c
  14. +38 −0 transient_heap.h
  15. +15 −15 vm_args.c
  16. +1 −1 vm_eval.c
  17. +1 −1 vm_insnhelper.c
398 array.c

Large diffs are not rendered by default.

Oops, something went wrong.
@@ -130,6 +130,7 @@ COMMONOBJS = array.$(OBJEXT) \
thread.$(OBJEXT) \
time.$(OBJEXT) \
transcode.$(OBJEXT) \
transient_heap.$(OBJEXT) \
util.$(OBJEXT) \
variable.$(OBJEXT) \
version.$(OBJEXT) \
@@ -2898,6 +2899,15 @@ transcode.$(OBJEXT): {$(VPATH)}st.h
transcode.$(OBJEXT): {$(VPATH)}subst.h
transcode.$(OBJEXT): {$(VPATH)}transcode.c
transcode.$(OBJEXT): {$(VPATH)}transcode_data.h
transient_heap.$(OBJEXT): $(hdrdir)/ruby/ruby.h
transient_heap.$(OBJEXT): $(top_srcdir)/include/ruby.h
transient_heap.$(OBJEXT): {$(VPATH)}debug_counter.h
transient_heap.$(OBJEXT): {$(VPATH)}gc.h
transient_heap.$(OBJEXT): {$(VPATH)}internal.h
transient_heap.$(OBJEXT): {$(VPATH)}ruby_assert.h
transient_heap.$(OBJEXT): {$(VPATH)}transient_heap.c
transient_heap.$(OBJEXT): {$(VPATH)}transient_heap.h
transient_heap.$(OBJEXT): {$(VPATH)}vm_debug.h
util.$(OBJEXT): $(hdrdir)/ruby/ruby.h
util.$(OBJEXT): $(top_srcdir)/include/ruby.h
util.$(OBJEXT): {$(VPATH)}config.h
@@ -1649,7 +1649,7 @@ iseq_set_arguments(rb_iseq_t *iseq, LINK_ANCHOR *const optargs, const NODE *cons
opt_table = ALLOC_N(VALUE, i+1);
MEMCPY(opt_table, RARRAY_CONST_PTR(labels), VALUE, i+1);
MEMCPY(opt_table, RARRAY_CONST_PTR_TRANSIENT(labels), VALUE, i+1);
for (j = 0; j < i+1; j++) {
opt_table[j] &= ~1;
}
@@ -2297,14 +2297,14 @@ iseq_set_exception_table(rb_iseq_t *iseq)
struct iseq_catch_table_entry *entry;
tlen = (int)RARRAY_LEN(ISEQ_COMPILE_DATA(iseq)->catch_table_ary);
tptr = RARRAY_CONST_PTR(ISEQ_COMPILE_DATA(iseq)->catch_table_ary);
tptr = RARRAY_CONST_PTR_TRANSIENT(ISEQ_COMPILE_DATA(iseq)->catch_table_ary);
if (tlen > 0) {
struct iseq_catch_table *table = xmalloc(iseq_catch_table_bytes(tlen));
table->size = tlen;
for (i = 0; i < table->size; i++) {
ptr = RARRAY_CONST_PTR(tptr[i]);
ptr = RARRAY_CONST_PTR_TRANSIENT(tptr[i]);
entry = &table->entries[i];
entry->type = (enum catch_type)(ptr[0] & 0xffff);
entry->start = label_get_position((LABEL *)(ptr[1] & ~1));
@@ -219,6 +219,11 @@ RB_DEBUG_COUNTER(heap_xmalloc)
RB_DEBUG_COUNTER(heap_xrealloc)
RB_DEBUG_COUNTER(heap_xfree)
/* transient_heap */
RB_DEBUG_COUNTER(theap_alloc)
RB_DEBUG_COUNTER(theap_alloc_fail)
RB_DEBUG_COUNTER(theap_evacuate)
/* load (not implemented yet) */
/*
RB_DEBUG_COUNTER(load_files)
6 enum.c
@@ -14,6 +14,7 @@
#include "ruby/util.h"
#include "id.h"
#include "symbol.h"
#include "transient_heap.h"
#include <assert.h>
@@ -1171,9 +1172,10 @@ enum_sort_by(VALUE obj)
rb_ary_concat(ary, buf);
}
if (RARRAY_LEN(ary) > 2) {
rb_ary_transient_heap_evacuate(ary, TRUE); /* should be malloc heap */
RARRAY_PTR_USE(ary, ptr,
ruby_qsort(ptr, RARRAY_LEN(ary)/2, 2*sizeof(VALUE),
sort_by_cmp, (void *)ary));
ruby_qsort(ptr, RARRAY_LEN(ary)/2, 2*sizeof(VALUE),
sort_by_cmp, (void *)ary));
}
if (RBASIC(ary)->klass) {
rb_raise(rb_eRuntimeError, "sort_by reentered");
89 gc.c
@@ -35,6 +35,7 @@
#include <sys/types.h>
#include "ruby_assert.h"
#include "debug_counter.h"
#include "transient_heap.h"
#include "mjit.h"
#undef rb_data_object_wrap
@@ -845,8 +846,6 @@ static void rb_objspace_call_finalizer(rb_objspace_t *objspace);
static VALUE define_final0(VALUE obj, VALUE block);
static void negative_size_allocation_error(const char *);
static void *aligned_malloc(size_t, size_t);
static void aligned_free(void *);
static void init_mark_stack(mark_stack_t *stack);
@@ -1190,6 +1189,7 @@ RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, struct heap_page *pag
{
MARK_IN_BITMAP(&page->uncollectible_bits[0], obj);
objspace->rgengc.old_objects++;
rb_transient_heap_promote(obj);
#if RGENGC_PROFILE >= 2
objspace->profile.total_promoted_count++;
@@ -1486,7 +1486,7 @@ heap_page_free(rb_objspace_t *objspace, struct heap_page *page)
{
heap_allocated_pages--;
objspace->profile.total_freed_pages++;
aligned_free(GET_PAGE_BODY(page->start));
rb_aligned_free(GET_PAGE_BODY(page->start));
free(page);
}
@@ -1524,15 +1524,15 @@ heap_page_allocate(rb_objspace_t *objspace)
int limit = HEAP_PAGE_OBJ_LIMIT;
/* assign heap_page body (contains heap_page_header and RVALUEs) */
page_body = (struct heap_page_body *)aligned_malloc(HEAP_PAGE_ALIGN, HEAP_PAGE_SIZE);
page_body = (struct heap_page_body *)rb_aligned_malloc(HEAP_PAGE_ALIGN, HEAP_PAGE_SIZE);
if (page_body == 0) {
rb_memerror();
}
/* assign heap_page entry */
page = (struct heap_page *)calloc(1, sizeof(struct heap_page));
if (page == 0) {
aligned_free(page_body);
rb_aligned_free(page_body);
rb_memerror();
}
@@ -4515,6 +4515,7 @@ static void
gc_mark_ptr(rb_objspace_t *objspace, VALUE obj)
{
if (LIKELY(objspace->mark_func_data == NULL)) {
if (RB_TYPE_P(obj, T_NONE)) rb_bug("...");
rgengc_check_relation(objspace, obj);
if (!gc_mark_set(objspace, obj)) return; /* already marked */
gc_aging(objspace, obj);
@@ -4672,14 +4673,22 @@ gc_mark_children(rb_objspace_t *objspace, VALUE obj)
case T_ARRAY:
if (FL_TEST(obj, ELTS_SHARED)) {
gc_mark(objspace, any->as.array.as.heap.aux.shared);
VALUE root = any->as.array.as.heap.aux.shared;
gc_mark(objspace, root);
}
else {
long i, len = RARRAY_LEN(obj);
const VALUE *ptr = RARRAY_CONST_PTR(obj);
const VALUE *ptr = RARRAY_CONST_PTR_TRANSIENT(obj);
for (i=0; i < len; i++) {
gc_mark(objspace, *ptr++);
gc_mark(objspace, ptr[i]);
}
if (objspace->mark_func_data == NULL) {
if (!FL_TEST_RAW(obj, RARRAY_EMBED_FLAG) &&
RARRAY_TRANSIENT_P(obj)) {
rb_transient_heap_mark(obj, ptr);
}
}
}
break;
@@ -5455,6 +5464,13 @@ rb_gc_verify_internal_consistency(void)
gc_verify_internal_consistency(Qnil);
}
static VALUE
gc_verify_transient_heap_internal_consistency(VALUE dmy)
{
rb_transient_heap_verify();
return Qnil;
}
/* marks */
static void
@@ -5671,6 +5687,8 @@ gc_marks_finish(rb_objspace_t *objspace)
#endif
}
rb_transient_heap_finish_marking();
gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_END_MARK, 0);
return TRUE;
@@ -6562,6 +6580,7 @@ gc_start(rb_objspace_t *objspace, int reason)
objspace->profile.heap_used_at_gc_start = heap_allocated_pages;
gc_prof_setup_new_record(objspace, reason);
gc_reset_malloc_info(objspace);
rb_transient_heap_start_marking(do_full_mark);
gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_START, 0 /* TODO: pass minor/immediate flag? */);
GC_ASSERT(during_gc);
@@ -7812,8 +7831,8 @@ rb_memerror(void)
EC_JUMP_TAG(ec, TAG_RAISE);
}
static void *
aligned_malloc(size_t alignment, size_t size)
void *
rb_aligned_malloc(size_t alignment, size_t size)
{
void *res;
@@ -7846,8 +7865,8 @@ aligned_malloc(size_t alignment, size_t size)
return res;
}
static void
aligned_free(void *ptr)
void
rb_aligned_free(void *ptr)
{
#if defined __MINGW32__
__mingw_aligned_free(ptr);
@@ -9551,13 +9570,21 @@ rb_raw_obj_info(char *buff, const int buff_size, VALUE obj)
#if USE_RGENGC
const int age = RVALUE_FLAGS_AGE(RBASIC(obj)->flags);
snprintf(buff, buff_size, "%p [%d%s%s%s%s] %s",
(void *)obj, age,
C(RVALUE_UNCOLLECTIBLE_BITMAP(obj), "L"),
C(RVALUE_MARK_BITMAP(obj), "M"),
C(RVALUE_MARKING_BITMAP(obj), "R"),
C(RVALUE_WB_UNPROTECTED_BITMAP(obj), "U"),
obj_type_name(obj));
if (is_pointer_to_heap(&rb_objspace, (void *)obj)) {
snprintf(buff, buff_size, "%p [%d%s%s%s%s] %s",
(void *)obj, age,
C(RVALUE_UNCOLLECTIBLE_BITMAP(obj), "L"),
C(RVALUE_MARK_BITMAP(obj), "M"),
C(RVALUE_MARKING_BITMAP(obj), "R"),
C(RVALUE_WB_UNPROTECTED_BITMAP(obj), "U"),
obj_type_name(obj));
}
else {
/* fake */
snprintf(buff, buff_size, "%p [%dXXXX] %s",
(void *)obj, age,
obj_type_name(obj));
}
#else
snprintf(buff, buff_size, "%p [%s] %s",
(void *)obj,
@@ -9587,10 +9614,25 @@ rb_raw_obj_info(char *buff, const int buff_size, VALUE obj)
UNEXPECTED_NODE(rb_raw_obj_info);
break;
case T_ARRAY:
snprintf(buff, buff_size, "%s [%s%s] len: %d", buff,
C(ARY_EMBED_P(obj), "E"),
C(ARY_SHARED_P(obj), "S"),
(int)RARRAY_LEN(obj));
if (FL_TEST(obj, ELTS_SHARED)) {
snprintf(buff, buff_size, "%s shared -> %s", buff,
rb_obj_info(RARRAY(obj)->as.heap.aux.shared));
}
else if (FL_TEST(obj, RARRAY_EMBED_FLAG)) {
snprintf(buff, buff_size, "%s [%s%s] len: %d (embed)", buff,
C(ARY_EMBED_P(obj), "E"),
C(ARY_SHARED_P(obj), "S"),
(int)RARRAY_LEN(obj));
}
else {
snprintf(buff, buff_size, "%s [%s%s%s] len: %d, capa:%d ptr:%p", buff,
C(ARY_EMBED_P(obj), "E"),
C(ARY_SHARED_P(obj), "S"),
C(RARRAY_TRANSIENT_P(obj), "T"),
(int)RARRAY_LEN(obj),
ARY_EMBED_P(obj) ? -1 : (int)RARRAY(obj)->as.heap.aux.capa,
RARRAY_CONST_PTR_TRANSIENT(obj));
}
break;
case T_STRING: {
snprintf(buff, buff_size, "%s %s", buff, RSTRING_PTR(obj));
@@ -9954,6 +9996,7 @@ Init_GC(void)
/* internal methods */
rb_define_singleton_method(rb_mGC, "verify_internal_consistency", gc_verify_internal_consistency, 0);
rb_define_singleton_method(rb_mGC, "verify_transient_heap_internal_consistency", gc_verify_transient_heap_internal_consistency, 0);
#if MALLOC_ALLOCATED_SIZE
rb_define_singleton_method(rb_mGC, "malloc_allocated_size", gc_malloc_allocated_size, 0);
rb_define_singleton_method(rb_mGC, "malloc_allocations", gc_malloc_allocations, 0);
@@ -1020,12 +1020,15 @@ enum ruby_rarray_flags {
RARRAY_EMBED_LEN_MASK = (RUBY_FL_USER4|RUBY_FL_USER3),
RARRAY_EMBED_LEN_SHIFT = (RUBY_FL_USHIFT+3),
RARRAY_TRANSIENT_FLAG = RUBY_FL_USER13,
RARRAY_ENUM_END
};
#define RARRAY_EMBED_FLAG (VALUE)RARRAY_EMBED_FLAG
#define RARRAY_EMBED_LEN_MASK (VALUE)RARRAY_EMBED_LEN_MASK
#define RARRAY_EMBED_LEN_MAX RARRAY_EMBED_LEN_MAX
#define RARRAY_EMBED_LEN_SHIFT RARRAY_EMBED_LEN_SHIFT
#define RARRAY_TRANSIENT_FLAG RARRAY_TRANSIENT_FLAG
struct RArray {
struct RBasic basic;
union {
@@ -1046,9 +1049,14 @@ struct RArray {
#define RARRAY_LEN(a) rb_array_len(a)
#define RARRAY_LENINT(ary) rb_long2int(RARRAY_LEN(ary))
#define RARRAY_CONST_PTR(a) rb_array_const_ptr(a)
#define RARRAY_CONST_PTR_TRANSIENT(a) rb_array_const_ptr_transient(a)
#define RARRAY_TRANSIENT_P(ary) FL_TEST_RAW((ary), RARRAY_TRANSIENT_FLAG)
VALUE *rb_ary_ptr_use_start(VALUE ary);
void rb_ary_ptr_use_end(VALUE ary);
#define RARRAY_PTR_USE_START(a) ((VALUE *)RARRAY_CONST_PTR(a))
#define RARRAY_PTR_USE_END(a) /* */
#define RARRAY_PTR_USE_START(a) rb_ary_ptr_use_start(a)
#define RARRAY_PTR_USE_END(a) rb_ary_ptr_use_end(a)
#define RARRAY_PTR_USE(ary, ptr_name, expr) do { \
const VALUE _ary = (ary); \
@@ -1057,11 +1065,12 @@ struct RArray {
RARRAY_PTR_USE_END(_ary); \
} while (0)
#define RARRAY_AREF(a, i) (RARRAY_CONST_PTR(a)[i])
#define RARRAY_AREF(a, i) (RARRAY_CONST_PTR_TRANSIENT(a)[i])
#define RARRAY_ASET(a, i, v) do { \
const VALUE _ary = (a); \
const VALUE _v = (v); \
VALUE *ptr = (VALUE *)RARRAY_PTR_USE_START(_ary); \
RB_OBJ_WRITE(_ary, &ptr[i], (v)); \
RB_OBJ_WRITE(_ary, &ptr[i], _v); \
RARRAY_PTR_USE_END(_ary); \
} while (0)
@@ -2110,12 +2119,23 @@ rb_array_len(VALUE a)
#endif
static inline const VALUE *
rb_array_const_ptr(VALUE a)
rb_array_const_ptr_transient(VALUE a)
{
return FIX_CONST_VALUE_PTR((RBASIC(a)->flags & RARRAY_EMBED_FLAG) ?
RARRAY(a)->as.ary : RARRAY(a)->as.heap.ptr);
}
void rb_ary_detransient(VALUE a);
static inline const VALUE *
rb_array_const_ptr(VALUE a)
{
if (RARRAY_TRANSIENT_P(a)) {
rb_ary_detransient(a);
}
return rb_array_const_ptr_transient(a);
}
#if defined(EXTLIB) && defined(USE_DLN_A_OUT)
/* hook for external modules */
static char *dln_libs_to_be_linked[] = { EXTLIB, 0 };
@@ -16,6 +16,7 @@
void
rb_call_inits(void)
{
CALL(TransientHeap);
CALL(Method);
CALL(RandomSeedCore);
CALL(sym);
@@ -524,7 +524,7 @@ newhashfromarray
{
VM_ASSERT(num * 2 == (rb_num_t)RARRAY_LEN(ary));
hash = rb_hash_new_with_size(num);
rb_hash_bulk_insert(num * 2, RARRAY_CONST_PTR(ary), hash);
rb_hash_bulk_insert(num * 2, RARRAY_CONST_PTR_TRANSIENT(ary), hash);
}
/* put new Range object.(Range.new(low, high, flag)) */
Oops, something went wrong.

0 comments on commit 312b105

Please sign in to comment.