Skip to content
Permalink
38a4f617de
Switch branches/tags

Name already in use

A tag already exists with the provided branch name. Many Git commands accept both tag and branch names, so creating this branch may cause unexpected behavior. Are you sure you want to create this branch?
Go to file
 
 
Cannot retrieve contributors at this time
12107 lines (10494 sloc) 327 KB
/**********************************************************************
gc.c -
$Author$
created at: Tue Oct 5 09:44:46 JST 1993
Copyright (C) 1993-2007 Yukihiro Matsumoto
Copyright (C) 2000 Network Applied Communication Laboratory, Inc.
Copyright (C) 2000 Information-technology Promotion Agency, Japan
**********************************************************************/
#define rb_data_object_alloc rb_data_object_alloc
#define rb_data_typed_object_alloc rb_data_typed_object_alloc
#include "ruby/internal/config.h"
#ifdef _WIN32
# include "ruby/ruby.h"
#endif
#include <setjmp.h>
#include <stdarg.h>
#include <stdio.h>
#ifndef HAVE_MALLOC_USABLE_SIZE
# ifdef _WIN32
# define HAVE_MALLOC_USABLE_SIZE
# define malloc_usable_size(a) _msize(a)
# elif defined HAVE_MALLOC_SIZE
# define HAVE_MALLOC_USABLE_SIZE
# define malloc_usable_size(a) malloc_size(a)
# endif
#endif
#ifdef HAVE_MALLOC_USABLE_SIZE
# ifdef RUBY_ALTERNATIVE_MALLOC_HEADER
# include RUBY_ALTERNATIVE_MALLOC_HEADER
# elif HAVE_MALLOC_H
# include <malloc.h>
# elif defined(HAVE_MALLOC_NP_H)
# include <malloc_np.h>
# elif defined(HAVE_MALLOC_MALLOC_H)
# include <malloc/malloc.h>
# endif
#endif
#ifdef HAVE_SYS_TIME_H
# include <sys/time.h>
#endif
#ifdef HAVE_SYS_RESOURCE_H
# include <sys/resource.h>
#endif
#if defined _WIN32 || defined __CYGWIN__
# include <windows.h>
#elif defined(HAVE_POSIX_MEMALIGN)
#elif defined(HAVE_MEMALIGN)
# include <malloc.h>
#endif
#include <sys/types.h>
#include "constant.h"
#include "debug_counter.h"
#include "eval_intern.h"
#include "gc.h"
#include "id_table.h"
#include "internal.h"
#include "internal/class.h"
#include "internal/complex.h"
#include "internal/cont.h"
#include "internal/error.h"
#include "internal/eval.h"
#include "internal/gc.h"
#include "internal/hash.h"
#include "internal/imemo.h"
#include "internal/io.h"
#include "internal/numeric.h"
#include "internal/object.h"
#include "internal/proc.h"
#include "internal/rational.h"
#include "internal/sanitizers.h"
#include "internal/struct.h"
#include "internal/symbol.h"
#include "internal/thread.h"
#include "internal/variable.h"
#include "internal/warnings.h"
#include "mjit.h"
#include "probes.h"
#include "regint.h"
#include "ruby/debug.h"
#include "ruby/io.h"
#include "ruby/re.h"
#include "ruby/st.h"
#include "ruby/thread.h"
#include "ruby/util.h"
#include "ruby_assert.h"
#include "ruby_atomic.h"
#include "symbol.h"
#include "transient_heap.h"
#include "vm_core.h"
#include "vm_callinfo.h"
#include "builtin.h"
#define rb_setjmp(env) RUBY_SETJMP(env)
#define rb_jmp_buf rb_jmpbuf_t
#undef rb_data_object_wrap
static inline struct rbimpl_size_mul_overflow_tag
size_add_overflow(size_t x, size_t y)
{
size_t z;
bool p;
#if 0
#elif __has_builtin(__builtin_add_overflow)
p = __builtin_add_overflow(x, y, &z);
#elif defined(DSIZE_T)
RB_GNUC_EXTENSION DSIZE_T dx = x;
RB_GNUC_EXTENSION DSIZE_T dy = y;
RB_GNUC_EXTENSION DSIZE_T dz = dx + dy;
p = dz > SIZE_MAX;
z = (size_t)dz;
#else
z = x + y;
p = z < y;
#endif
return (struct rbimpl_size_mul_overflow_tag) { p, z, };
}
static inline struct rbimpl_size_mul_overflow_tag
size_mul_add_overflow(size_t x, size_t y, size_t z) /* x * y + z */
{
struct rbimpl_size_mul_overflow_tag t = rbimpl_size_mul_overflow(x, y);
struct rbimpl_size_mul_overflow_tag u = size_add_overflow(t.right, z);
return (struct rbimpl_size_mul_overflow_tag) { t.left || u.left, u.right };
}
static inline struct rbimpl_size_mul_overflow_tag
size_mul_add_mul_overflow(size_t x, size_t y, size_t z, size_t w) /* x * y + z * w */
{
struct rbimpl_size_mul_overflow_tag t = rbimpl_size_mul_overflow(x, y);
struct rbimpl_size_mul_overflow_tag u = rbimpl_size_mul_overflow(z, w);
struct rbimpl_size_mul_overflow_tag v = size_add_overflow(t.right, u.right);
return (struct rbimpl_size_mul_overflow_tag) { t.left || u.left || v.left, v.right };
}
PRINTF_ARGS(NORETURN(static void gc_raise(VALUE, const char*, ...)), 2, 3);
static inline size_t
size_mul_or_raise(size_t x, size_t y, VALUE exc)
{
struct rbimpl_size_mul_overflow_tag t = rbimpl_size_mul_overflow(x, y);
if (LIKELY(!t.left)) {
return t.right;
}
else if (rb_during_gc()) {
rb_memerror(); /* or...? */
}
else {
gc_raise(
exc,
"integer overflow: %"PRIuSIZE
" * %"PRIuSIZE
" > %"PRIuSIZE,
x, y, SIZE_MAX);
}
}
size_t
rb_size_mul_or_raise(size_t x, size_t y, VALUE exc)
{
return size_mul_or_raise(x, y, exc);
}
static inline size_t
size_mul_add_or_raise(size_t x, size_t y, size_t z, VALUE exc)
{
struct rbimpl_size_mul_overflow_tag t = size_mul_add_overflow(x, y, z);
if (LIKELY(!t.left)) {
return t.right;
}
else if (rb_during_gc()) {
rb_memerror(); /* or...? */
}
else {
gc_raise(
exc,
"integer overflow: %"PRIuSIZE
" * %"PRIuSIZE
" + %"PRIuSIZE
" > %"PRIuSIZE,
x, y, z, SIZE_MAX);
}
}
size_t
rb_size_mul_add_or_raise(size_t x, size_t y, size_t z, VALUE exc)
{
return size_mul_add_or_raise(x, y, z, exc);
}
static inline size_t
size_mul_add_mul_or_raise(size_t x, size_t y, size_t z, size_t w, VALUE exc)
{
struct rbimpl_size_mul_overflow_tag t = size_mul_add_mul_overflow(x, y, z, w);
if (LIKELY(!t.left)) {
return t.right;
}
else if (rb_during_gc()) {
rb_memerror(); /* or...? */
}
else {
gc_raise(
exc,
"integer overflow: %"PRIdSIZE
" * %"PRIdSIZE
" + %"PRIdSIZE
" * %"PRIdSIZE
" > %"PRIdSIZE,
x, y, z, w, SIZE_MAX);
}
}
#if defined(HAVE_RB_GC_GUARDED_PTR_VAL) && HAVE_RB_GC_GUARDED_PTR_VAL
/* trick the compiler into thinking a external signal handler uses this */
volatile VALUE rb_gc_guarded_val;
volatile VALUE *
rb_gc_guarded_ptr_val(volatile VALUE *ptr, VALUE val)
{
rb_gc_guarded_val = val;
return ptr;
}
#endif
#ifndef GC_HEAP_INIT_SLOTS
#define GC_HEAP_INIT_SLOTS 10000
#endif
#ifndef GC_HEAP_FREE_SLOTS
#define GC_HEAP_FREE_SLOTS 4096
#endif
#ifndef GC_HEAP_GROWTH_FACTOR
#define GC_HEAP_GROWTH_FACTOR 1.8
#endif
#ifndef GC_HEAP_GROWTH_MAX_SLOTS
#define GC_HEAP_GROWTH_MAX_SLOTS 0 /* 0 is disable */
#endif
#ifndef GC_HEAP_OLDOBJECT_LIMIT_FACTOR
#define GC_HEAP_OLDOBJECT_LIMIT_FACTOR 2.0
#endif
#ifndef GC_HEAP_FREE_SLOTS_MIN_RATIO
#define GC_HEAP_FREE_SLOTS_MIN_RATIO 0.20
#endif
#ifndef GC_HEAP_FREE_SLOTS_GOAL_RATIO
#define GC_HEAP_FREE_SLOTS_GOAL_RATIO 0.40
#endif
#ifndef GC_HEAP_FREE_SLOTS_MAX_RATIO
#define GC_HEAP_FREE_SLOTS_MAX_RATIO 0.65
#endif
#ifndef GC_MALLOC_LIMIT_MIN
#define GC_MALLOC_LIMIT_MIN (16 * 1024 * 1024 /* 16MB */)
#endif
#ifndef GC_MALLOC_LIMIT_MAX
#define GC_MALLOC_LIMIT_MAX (32 * 1024 * 1024 /* 32MB */)
#endif
#ifndef GC_MALLOC_LIMIT_GROWTH_FACTOR
#define GC_MALLOC_LIMIT_GROWTH_FACTOR 1.4
#endif
#ifndef GC_OLDMALLOC_LIMIT_MIN
#define GC_OLDMALLOC_LIMIT_MIN (16 * 1024 * 1024 /* 16MB */)
#endif
#ifndef GC_OLDMALLOC_LIMIT_GROWTH_FACTOR
#define GC_OLDMALLOC_LIMIT_GROWTH_FACTOR 1.2
#endif
#ifndef GC_OLDMALLOC_LIMIT_MAX
#define GC_OLDMALLOC_LIMIT_MAX (128 * 1024 * 1024 /* 128MB */)
#endif
#ifndef PRINT_MEASURE_LINE
#define PRINT_MEASURE_LINE 0
#endif
#ifndef PRINT_ENTER_EXIT_TICK
#define PRINT_ENTER_EXIT_TICK 0
#endif
#ifndef PRINT_ROOT_TICKS
#define PRINT_ROOT_TICKS 0
#endif
#define USE_TICK_T (PRINT_ENTER_EXIT_TICK || PRINT_MEASURE_LINE || PRINT_ROOT_TICKS)
#define TICK_TYPE 1
typedef struct {
size_t heap_init_slots;
size_t heap_free_slots;
double growth_factor;
size_t growth_max_slots;
double heap_free_slots_min_ratio;
double heap_free_slots_goal_ratio;
double heap_free_slots_max_ratio;
double oldobject_limit_factor;
size_t malloc_limit_min;
size_t malloc_limit_max;
double malloc_limit_growth_factor;
size_t oldmalloc_limit_min;
size_t oldmalloc_limit_max;
double oldmalloc_limit_growth_factor;
VALUE gc_stress;
} ruby_gc_params_t;
static ruby_gc_params_t gc_params = {
GC_HEAP_INIT_SLOTS,
GC_HEAP_FREE_SLOTS,
GC_HEAP_GROWTH_FACTOR,
GC_HEAP_GROWTH_MAX_SLOTS,
GC_HEAP_FREE_SLOTS_MIN_RATIO,
GC_HEAP_FREE_SLOTS_GOAL_RATIO,
GC_HEAP_FREE_SLOTS_MAX_RATIO,
GC_HEAP_OLDOBJECT_LIMIT_FACTOR,
GC_MALLOC_LIMIT_MIN,
GC_MALLOC_LIMIT_MAX,
GC_MALLOC_LIMIT_GROWTH_FACTOR,
GC_OLDMALLOC_LIMIT_MIN,
GC_OLDMALLOC_LIMIT_MAX,
GC_OLDMALLOC_LIMIT_GROWTH_FACTOR,
FALSE,
};
/* GC_DEBUG:
* enable to embed GC debugging information.
*/
#ifndef GC_DEBUG
#define GC_DEBUG 0
#endif
/* RGENGC_DEBUG:
* 1: basic information
* 2: remember set operation
* 3: mark
* 4:
* 5: sweep
*/
#ifndef RGENGC_DEBUG
#ifdef RUBY_DEVEL
#define RGENGC_DEBUG -1
#else
#define RGENGC_DEBUG 0
#endif
#endif
#if RGENGC_DEBUG < 0 && !defined(_MSC_VER)
# define RGENGC_DEBUG_ENABLED(level) (-(RGENGC_DEBUG) >= (level) && ruby_rgengc_debug >= (level))
#else
# define RGENGC_DEBUG_ENABLED(level) ((RGENGC_DEBUG) >= (level))
#endif
int ruby_rgengc_debug;
/* RGENGC_CHECK_MODE
* 0: disable all assertions
* 1: enable assertions (to debug RGenGC)
* 2: enable internal consistency check at each GC (for debugging)
* 3: enable internal consistency check at each GC steps (for debugging)
* 4: enable liveness check
* 5: show all references
*/
#ifndef RGENGC_CHECK_MODE
#define RGENGC_CHECK_MODE 0
#endif
// Note: using RUBY_ASSERT_WHEN() extend a macro in expr (info by nobu).
#define GC_ASSERT(expr) RUBY_ASSERT_MESG_WHEN(RGENGC_CHECK_MODE > 0, expr, #expr)
/* RGENGC_OLD_NEWOBJ_CHECK
* 0: disable all assertions
* >0: make a OLD object when new object creation.
*
* Make one OLD object per RGENGC_OLD_NEWOBJ_CHECK WB protected objects creation.
*/
#ifndef RGENGC_OLD_NEWOBJ_CHECK
#define RGENGC_OLD_NEWOBJ_CHECK 0
#endif
/* RGENGC_PROFILE
* 0: disable RGenGC profiling
* 1: enable profiling for basic information
* 2: enable profiling for each types
*/
#ifndef RGENGC_PROFILE
#define RGENGC_PROFILE 0
#endif
/* RGENGC_ESTIMATE_OLDMALLOC
* Enable/disable to estimate increase size of malloc'ed size by old objects.
* If estimation exceeds threshold, then will invoke full GC.
* 0: disable estimation.
* 1: enable estimation.
*/
#ifndef RGENGC_ESTIMATE_OLDMALLOC
#define RGENGC_ESTIMATE_OLDMALLOC 1
#endif
/* RGENGC_FORCE_MAJOR_GC
* Force major/full GC if this macro is not 0.
*/
#ifndef RGENGC_FORCE_MAJOR_GC
#define RGENGC_FORCE_MAJOR_GC 0
#endif
#ifndef GC_PROFILE_MORE_DETAIL
#define GC_PROFILE_MORE_DETAIL 0
#endif
#ifndef GC_PROFILE_DETAIL_MEMORY
#define GC_PROFILE_DETAIL_MEMORY 0
#endif
#ifndef GC_ENABLE_INCREMENTAL_MARK
#define GC_ENABLE_INCREMENTAL_MARK USE_RINCGC
#endif
#ifndef GC_ENABLE_LAZY_SWEEP
#define GC_ENABLE_LAZY_SWEEP 1
#endif
#ifndef CALC_EXACT_MALLOC_SIZE
#define CALC_EXACT_MALLOC_SIZE USE_GC_MALLOC_OBJ_INFO_DETAILS
#endif
#if defined(HAVE_MALLOC_USABLE_SIZE) || CALC_EXACT_MALLOC_SIZE > 0
#ifndef MALLOC_ALLOCATED_SIZE
#define MALLOC_ALLOCATED_SIZE 0
#endif
#else
#define MALLOC_ALLOCATED_SIZE 0
#endif
#ifndef MALLOC_ALLOCATED_SIZE_CHECK
#define MALLOC_ALLOCATED_SIZE_CHECK 0
#endif
#ifndef GC_DEBUG_STRESS_TO_CLASS
#define GC_DEBUG_STRESS_TO_CLASS 0
#endif
#ifndef RGENGC_OBJ_INFO
#define RGENGC_OBJ_INFO (RGENGC_DEBUG | RGENGC_CHECK_MODE)
#endif
typedef enum {
GPR_FLAG_NONE = 0x000,
/* major reason */
GPR_FLAG_MAJOR_BY_NOFREE = 0x001,
GPR_FLAG_MAJOR_BY_OLDGEN = 0x002,
GPR_FLAG_MAJOR_BY_SHADY = 0x004,
GPR_FLAG_MAJOR_BY_FORCE = 0x008,
#if RGENGC_ESTIMATE_OLDMALLOC
GPR_FLAG_MAJOR_BY_OLDMALLOC = 0x020,
#endif
GPR_FLAG_MAJOR_MASK = 0x0ff,
/* gc reason */
GPR_FLAG_NEWOBJ = 0x100,
GPR_FLAG_MALLOC = 0x200,
GPR_FLAG_METHOD = 0x400,
GPR_FLAG_CAPI = 0x800,
GPR_FLAG_STRESS = 0x1000,
/* others */
GPR_FLAG_IMMEDIATE_SWEEP = 0x2000,
GPR_FLAG_HAVE_FINALIZE = 0x4000,
GPR_FLAG_IMMEDIATE_MARK = 0x8000,
GPR_FLAG_FULL_MARK = 0x10000,
GPR_DEFAULT_REASON =
(GPR_FLAG_FULL_MARK | GPR_FLAG_IMMEDIATE_MARK |
GPR_FLAG_IMMEDIATE_SWEEP | GPR_FLAG_CAPI),
} gc_profile_record_flag;
typedef struct gc_profile_record {
int flags;
double gc_time;
double gc_invoke_time;
size_t heap_total_objects;
size_t heap_use_size;
size_t heap_total_size;
#if GC_PROFILE_MORE_DETAIL
double gc_mark_time;
double gc_sweep_time;
size_t heap_use_pages;
size_t heap_live_objects;
size_t heap_free_objects;
size_t allocate_increase;
size_t allocate_limit;
double prepare_time;
size_t removing_objects;
size_t empty_objects;
#if GC_PROFILE_DETAIL_MEMORY
long maxrss;
long minflt;
long majflt;
#endif
#endif
#if MALLOC_ALLOCATED_SIZE
size_t allocated_size;
#endif
#if RGENGC_PROFILE > 0
size_t old_objects;
size_t remembered_normal_objects;
size_t remembered_shady_objects;
#endif
} gc_profile_record;
struct RMoved {
VALUE flags;
VALUE destination;
VALUE next;
};
#if defined(_MSC_VER) || defined(__CYGWIN__)
#pragma pack(push, 1) /* magic for reducing sizeof(RVALUE): 24 -> 20 */
#endif
typedef struct RVALUE {
union {
struct {
VALUE flags; /* always 0 for freed obj */
struct RVALUE *next;
} free;
struct RMoved moved;
struct RBasic basic;
struct RObject object;
struct RClass klass;
struct RFloat flonum;
struct RString string;
struct RArray array;
struct RRegexp regexp;
struct RHash hash;
struct RData data;
struct RTypedData typeddata;
struct RStruct rstruct;
struct RBignum bignum;
struct RFile file;
struct RMatch match;
struct RRational rational;
struct RComplex complex;
union {
rb_cref_t cref;
struct vm_svar svar;
struct vm_throw_data throw_data;
struct vm_ifunc ifunc;
struct MEMO memo;
struct rb_method_entry_struct ment;
const rb_iseq_t iseq;
rb_env_t env;
struct rb_imemo_tmpbuf_struct alloc;
rb_ast_t ast;
} imemo;
struct {
struct RBasic basic;
VALUE v1;
VALUE v2;
VALUE v3;
} values;
} as;
#if GC_DEBUG
const char *file;
int line;
#endif
} RVALUE;
#if defined(_MSC_VER) || defined(__CYGWIN__)
#pragma pack(pop)
#endif
typedef uintptr_t bits_t;
enum {
BITS_SIZE = sizeof(bits_t),
BITS_BITLENGTH = ( BITS_SIZE * CHAR_BIT )
};
#define popcount_bits rb_popcount_intptr
struct heap_page_header {
struct heap_page *page;
};
struct heap_page_body {
struct heap_page_header header;
/* char gap[]; */
/* RVALUE values[]; */
};
struct gc_list {
VALUE *varptr;
struct gc_list *next;
};
#define STACK_CHUNK_SIZE 500
typedef struct stack_chunk {
VALUE data[STACK_CHUNK_SIZE];
struct stack_chunk *next;
} stack_chunk_t;
typedef struct mark_stack {
stack_chunk_t *chunk;
stack_chunk_t *cache;
int index;
int limit;
size_t cache_size;
size_t unused_cache_size;
} mark_stack_t;
typedef struct rb_heap_struct {
RVALUE *freelist;
struct heap_page *free_pages;
struct heap_page *using_page;
struct list_head pages;
struct heap_page *sweeping_page; /* iterator for .pages */
#if GC_ENABLE_INCREMENTAL_MARK
struct heap_page *pooled_pages;
#endif
size_t total_pages; /* total page count in a heap */
size_t total_slots; /* total slot count (about total_pages * HEAP_PAGE_OBJ_LIMIT) */
} rb_heap_t;
enum gc_mode {
gc_mode_none,
gc_mode_marking,
gc_mode_sweeping
};
typedef struct rb_objspace {
struct {
size_t limit;
size_t increase;
#if MALLOC_ALLOCATED_SIZE
size_t allocated_size;
size_t allocations;
#endif
} malloc_params;
struct {
unsigned int mode : 2;
unsigned int immediate_sweep : 1;
unsigned int dont_gc : 1;
unsigned int dont_incremental : 1;
unsigned int during_gc : 1;
unsigned int during_compacting : 1;
unsigned int gc_stressful: 1;
unsigned int has_hook: 1;
unsigned int during_minor_gc : 1;
#if GC_ENABLE_INCREMENTAL_MARK
unsigned int during_incremental_marking : 1;
#endif
} flags;
rb_event_flag_t hook_events;
size_t total_allocated_objects;
VALUE next_object_id;
rb_heap_t eden_heap;
rb_heap_t tomb_heap; /* heap for zombies and ghosts */
struct {
rb_atomic_t finalizing;
} atomic_flags;
struct mark_func_data_struct {
void *data;
void (*mark_func)(VALUE v, void *data);
} *mark_func_data;
mark_stack_t mark_stack;
size_t marked_slots;
struct {
struct heap_page **sorted;
size_t allocated_pages;
size_t allocatable_pages;
size_t sorted_length;
RVALUE *range[2];
size_t freeable_pages;
/* final */
size_t final_slots;
VALUE deferred_final;
} heap_pages;
st_table *finalizer_table;
struct {
int run;
int latest_gc_info;
gc_profile_record *records;
gc_profile_record *current_record;
size_t next_index;
size_t size;
#if GC_PROFILE_MORE_DETAIL
double prepare_time;
#endif
double invoke_time;
size_t minor_gc_count;
size_t major_gc_count;
size_t compact_count;
#if RGENGC_PROFILE > 0
size_t total_generated_normal_object_count;
size_t total_generated_shady_object_count;
size_t total_shade_operation_count;
size_t total_promoted_count;
size_t total_remembered_normal_object_count;
size_t total_remembered_shady_object_count;
#if RGENGC_PROFILE >= 2
size_t generated_normal_object_count_types[RUBY_T_MASK];
size_t generated_shady_object_count_types[RUBY_T_MASK];
size_t shade_operation_count_types[RUBY_T_MASK];
size_t promoted_types[RUBY_T_MASK];
size_t remembered_normal_object_count_types[RUBY_T_MASK];
size_t remembered_shady_object_count_types[RUBY_T_MASK];
#endif
#endif /* RGENGC_PROFILE */
/* temporary profiling space */
double gc_sweep_start_time;
size_t total_allocated_objects_at_gc_start;
size_t heap_used_at_gc_start;
/* basic statistics */
size_t count;
size_t total_freed_objects;
size_t total_allocated_pages;
size_t total_freed_pages;
} profile;
struct gc_list *global_list;
VALUE gc_stress_mode;
struct {
VALUE parent_object;
int need_major_gc;
size_t last_major_gc;
size_t uncollectible_wb_unprotected_objects;
size_t uncollectible_wb_unprotected_objects_limit;
size_t old_objects;
size_t old_objects_limit;
#if RGENGC_ESTIMATE_OLDMALLOC
size_t oldmalloc_increase;
size_t oldmalloc_increase_limit;
#endif
#if RGENGC_CHECK_MODE >= 2
struct st_table *allrefs_table;
size_t error_count;
#endif
} rgengc;
struct {
size_t considered_count_table[T_MASK];
size_t moved_count_table[T_MASK];
} rcompactor;
#if GC_ENABLE_INCREMENTAL_MARK
struct {
size_t pooled_slots;
size_t step_slots;
} rincgc;
#endif
st_table *id_to_obj_tbl;
st_table *obj_to_id_tbl;
#if GC_DEBUG_STRESS_TO_CLASS
VALUE stress_to_class;
#endif
} rb_objspace_t;
/* default tiny heap size: 16KB */
#define HEAP_PAGE_ALIGN_LOG 14
#define CEILDIV(i, mod) (((i) + (mod) - 1)/(mod))
enum {
HEAP_PAGE_ALIGN = (1UL << HEAP_PAGE_ALIGN_LOG),
HEAP_PAGE_ALIGN_MASK = (~(~0UL << HEAP_PAGE_ALIGN_LOG)),
REQUIRED_SIZE_BY_MALLOC = (sizeof(size_t) * 5),
HEAP_PAGE_SIZE = (HEAP_PAGE_ALIGN - REQUIRED_SIZE_BY_MALLOC),
HEAP_PAGE_OBJ_LIMIT = (unsigned int)((HEAP_PAGE_SIZE - sizeof(struct heap_page_header))/sizeof(struct RVALUE)),
HEAP_PAGE_BITMAP_LIMIT = CEILDIV(CEILDIV(HEAP_PAGE_SIZE, sizeof(struct RVALUE)), BITS_BITLENGTH),
HEAP_PAGE_BITMAP_SIZE = (BITS_SIZE * HEAP_PAGE_BITMAP_LIMIT),
HEAP_PAGE_BITMAP_PLANES = 4 /* RGENGC: mark, unprotected, uncollectible, marking */
};
struct heap_page {
short total_slots;
short free_slots;
short pinned_slots;
short final_slots;
struct {
unsigned int before_sweep : 1;
unsigned int has_remembered_objects : 1;
unsigned int has_uncollectible_shady_objects : 1;
unsigned int in_tomb : 1;
} flags;
struct heap_page *free_next;
RVALUE *start;
RVALUE *freelist;
struct list_node page_node;
bits_t wb_unprotected_bits[HEAP_PAGE_BITMAP_LIMIT];
/* the following three bitmaps are cleared at the beginning of full GC */
bits_t mark_bits[HEAP_PAGE_BITMAP_LIMIT];
bits_t uncollectible_bits[HEAP_PAGE_BITMAP_LIMIT];
bits_t marking_bits[HEAP_PAGE_BITMAP_LIMIT];
/* If set, the object is not movable */
bits_t pinned_bits[HEAP_PAGE_BITMAP_LIMIT];
};
#define GET_PAGE_BODY(x) ((struct heap_page_body *)((bits_t)(x) & ~(HEAP_PAGE_ALIGN_MASK)))
#define GET_PAGE_HEADER(x) (&GET_PAGE_BODY(x)->header)
#define GET_HEAP_PAGE(x) (GET_PAGE_HEADER(x)->page)
#define NUM_IN_PAGE(p) (((bits_t)(p) & HEAP_PAGE_ALIGN_MASK)/sizeof(RVALUE))
#define BITMAP_INDEX(p) (NUM_IN_PAGE(p) / BITS_BITLENGTH )
#define BITMAP_OFFSET(p) (NUM_IN_PAGE(p) & (BITS_BITLENGTH-1))
#define BITMAP_BIT(p) ((bits_t)1 << BITMAP_OFFSET(p))
/* Bitmap Operations */
#define MARKED_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] & BITMAP_BIT(p))
#define MARK_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] = (bits)[BITMAP_INDEX(p)] | BITMAP_BIT(p))
#define CLEAR_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] = (bits)[BITMAP_INDEX(p)] & ~BITMAP_BIT(p))
/* getting bitmap */
#define GET_HEAP_MARK_BITS(x) (&GET_HEAP_PAGE(x)->mark_bits[0])
#define GET_HEAP_PINNED_BITS(x) (&GET_HEAP_PAGE(x)->pinned_bits[0])
#define GET_HEAP_UNCOLLECTIBLE_BITS(x) (&GET_HEAP_PAGE(x)->uncollectible_bits[0])
#define GET_HEAP_WB_UNPROTECTED_BITS(x) (&GET_HEAP_PAGE(x)->wb_unprotected_bits[0])
#define GET_HEAP_MARKING_BITS(x) (&GET_HEAP_PAGE(x)->marking_bits[0])
/* Aliases */
#define rb_objspace (*rb_objspace_of(GET_VM()))
#define rb_objspace_of(vm) ((vm)->objspace)
#define ruby_initial_gc_stress gc_params.gc_stress
VALUE *ruby_initial_gc_stress_ptr = &ruby_initial_gc_stress;
#define malloc_limit objspace->malloc_params.limit
#define malloc_increase objspace->malloc_params.increase
#define malloc_allocated_size objspace->malloc_params.allocated_size
#define heap_pages_sorted objspace->heap_pages.sorted
#define heap_allocated_pages objspace->heap_pages.allocated_pages
#define heap_pages_sorted_length objspace->heap_pages.sorted_length
#define heap_pages_lomem objspace->heap_pages.range[0]
#define heap_pages_himem objspace->heap_pages.range[1]
#define heap_allocatable_pages objspace->heap_pages.allocatable_pages
#define heap_pages_freeable_pages objspace->heap_pages.freeable_pages
#define heap_pages_final_slots objspace->heap_pages.final_slots
#define heap_pages_deferred_final objspace->heap_pages.deferred_final
#define heap_eden (&objspace->eden_heap)
#define heap_tomb (&objspace->tomb_heap)
#define dont_gc objspace->flags.dont_gc
#define during_gc objspace->flags.during_gc
#define finalizing objspace->atomic_flags.finalizing
#define finalizer_table objspace->finalizer_table
#define global_list objspace->global_list
#define ruby_gc_stressful objspace->flags.gc_stressful
#define ruby_gc_stress_mode objspace->gc_stress_mode
#if GC_DEBUG_STRESS_TO_CLASS
#define stress_to_class objspace->stress_to_class
#else
#define stress_to_class 0
#endif
static inline enum gc_mode
gc_mode_verify(enum gc_mode mode)
{
#if RGENGC_CHECK_MODE > 0
switch (mode) {
case gc_mode_none:
case gc_mode_marking:
case gc_mode_sweeping:
break;
default:
rb_bug("gc_mode_verify: unreachable (%d)", (int)mode);
}
#endif
return mode;
}
#define gc_mode(objspace) gc_mode_verify((enum gc_mode)(objspace)->flags.mode)
#define gc_mode_set(objspace, mode) ((objspace)->flags.mode = (unsigned int)gc_mode_verify(mode))
#define is_marking(objspace) (gc_mode(objspace) == gc_mode_marking)
#define is_sweeping(objspace) (gc_mode(objspace) == gc_mode_sweeping)
#define is_full_marking(objspace) ((objspace)->flags.during_minor_gc == FALSE)
#if GC_ENABLE_INCREMENTAL_MARK
#define is_incremental_marking(objspace) ((objspace)->flags.during_incremental_marking != FALSE)
#else
#define is_incremental_marking(objspace) FALSE
#endif
#if GC_ENABLE_INCREMENTAL_MARK
#define will_be_incremental_marking(objspace) ((objspace)->rgengc.need_major_gc != GPR_FLAG_NONE)
#else
#define will_be_incremental_marking(objspace) FALSE
#endif
#define has_sweeping_pages(heap) ((heap)->sweeping_page != 0)
#define is_lazy_sweeping(heap) (GC_ENABLE_LAZY_SWEEP && has_sweeping_pages(heap))
#if SIZEOF_LONG == SIZEOF_VOIDP
# define nonspecial_obj_id(obj) (VALUE)((SIGNED_VALUE)(obj)|FIXNUM_FLAG)
# define obj_id_to_ref(objid) ((objid) ^ FIXNUM_FLAG) /* unset FIXNUM_FLAG */
#elif SIZEOF_LONG_LONG == SIZEOF_VOIDP
# define nonspecial_obj_id(obj) LL2NUM((SIGNED_VALUE)(obj) / 2)
# define obj_id_to_ref(objid) (FIXNUM_P(objid) ? \
((objid) ^ FIXNUM_FLAG) : (NUM2PTR(objid) << 1))
#else
# error not supported
#endif
#define RANY(o) ((RVALUE*)(o))
struct RZombie {
struct RBasic basic;
VALUE next;
void (*dfree)(void *);
void *data;
};
#define RZOMBIE(o) ((struct RZombie *)(o))
#define nomem_error GET_VM()->special_exceptions[ruby_error_nomemory]
#if RUBY_MARK_FREE_DEBUG
int ruby_gc_debug_indent = 0;
#endif
VALUE rb_mGC;
int ruby_disable_gc = 0;
void rb_iseq_mark(const rb_iseq_t *iseq);
void rb_iseq_update_references(rb_iseq_t *iseq);
void rb_iseq_free(const rb_iseq_t *iseq);
size_t rb_iseq_memsize(const rb_iseq_t *iseq);
void rb_vm_update_references(void *ptr);
void rb_gcdebug_print_obj_condition(VALUE obj);
static VALUE define_final0(VALUE obj, VALUE block);
NORETURN(static void *gc_vraise(void *ptr));
NORETURN(static void gc_raise(VALUE exc, const char *fmt, ...));
NORETURN(static void negative_size_allocation_error(const char *));
static void init_mark_stack(mark_stack_t *stack);
static int ready_to_gc(rb_objspace_t *objspace);
static int garbage_collect(rb_objspace_t *, int reason);
static int gc_start(rb_objspace_t *objspace, int reason);
static void gc_rest(rb_objspace_t *objspace);
static inline void gc_enter(rb_objspace_t *objspace, const char *event);
static inline void gc_exit(rb_objspace_t *objspace, const char *event);
static void gc_marks(rb_objspace_t *objspace, int full_mark);
static void gc_marks_start(rb_objspace_t *objspace, int full);
static int gc_marks_finish(rb_objspace_t *objspace);
static void gc_marks_rest(rb_objspace_t *objspace);
static void gc_marks_step(rb_objspace_t *objspace, int slots);
static void gc_marks_continue(rb_objspace_t *objspace, rb_heap_t *heap);
static void gc_sweep(rb_objspace_t *objspace);
static void gc_sweep_start(rb_objspace_t *objspace);
static void gc_sweep_finish(rb_objspace_t *objspace);
static int gc_sweep_step(rb_objspace_t *objspace, rb_heap_t *heap);
static void gc_sweep_rest(rb_objspace_t *objspace);
static void gc_sweep_continue(rb_objspace_t *objspace, rb_heap_t *heap);
static inline void gc_mark(rb_objspace_t *objspace, VALUE ptr);
static inline void gc_pin(rb_objspace_t *objspace, VALUE ptr);
static inline void gc_mark_and_pin(rb_objspace_t *objspace, VALUE ptr);
static void gc_mark_ptr(rb_objspace_t *objspace, VALUE ptr);
NO_SANITIZE("memory", static void gc_mark_maybe(rb_objspace_t *objspace, VALUE ptr));
static void gc_mark_children(rb_objspace_t *objspace, VALUE ptr);
static int gc_mark_stacked_objects_incremental(rb_objspace_t *, size_t count);
static int gc_mark_stacked_objects_all(rb_objspace_t *);
static void gc_grey(rb_objspace_t *objspace, VALUE ptr);
static inline int gc_mark_set(rb_objspace_t *objspace, VALUE obj);
NO_SANITIZE("memory", static inline int is_pointer_to_heap(rb_objspace_t *objspace, void *ptr));
static void push_mark_stack(mark_stack_t *, VALUE);
static int pop_mark_stack(mark_stack_t *, VALUE *);
static size_t mark_stack_size(mark_stack_t *stack);
static void shrink_stack_chunk_cache(mark_stack_t *stack);
static size_t obj_memsize_of(VALUE obj, int use_all_types);
static void gc_verify_internal_consistency(rb_objspace_t *objspace);
static int gc_verify_heap_page(rb_objspace_t *objspace, struct heap_page *page, VALUE obj);
static int gc_verify_heap_pages(rb_objspace_t *objspace);
static void gc_stress_set(rb_objspace_t *objspace, VALUE flag);
static VALUE gc_disable_no_rest(rb_objspace_t *);
static double getrusage_time(void);
static inline void gc_prof_setup_new_record(rb_objspace_t *objspace, int reason);
static inline void gc_prof_timer_start(rb_objspace_t *);
static inline void gc_prof_timer_stop(rb_objspace_t *);
static inline void gc_prof_mark_timer_start(rb_objspace_t *);
static inline void gc_prof_mark_timer_stop(rb_objspace_t *);
static inline void gc_prof_sweep_timer_start(rb_objspace_t *);
static inline void gc_prof_sweep_timer_stop(rb_objspace_t *);
static inline void gc_prof_set_malloc_info(rb_objspace_t *);
static inline void gc_prof_set_heap_info(rb_objspace_t *);
#define TYPED_UPDATE_IF_MOVED(_objspace, _type, _thing) do { \
if (gc_object_moved_p(_objspace, (VALUE)_thing)) { \
*((_type *)(&_thing)) = (_type)RMOVED((_thing))->destination; \
} \
} while (0)
#define UPDATE_IF_MOVED(_objspace, _thing) TYPED_UPDATE_IF_MOVED(_objspace, VALUE, _thing)
#define gc_prof_record(objspace) (objspace)->profile.current_record
#define gc_prof_enabled(objspace) ((objspace)->profile.run && (objspace)->profile.current_record)
#ifdef HAVE_VA_ARGS_MACRO
# define gc_report(level, objspace, ...) \
if (!RGENGC_DEBUG_ENABLED(level)) {} else gc_report_body(level, objspace, __VA_ARGS__)
#else
# define gc_report if (!RGENGC_DEBUG_ENABLED(0)) {} else gc_report_body
#endif
PRINTF_ARGS(static void gc_report_body(int level, rb_objspace_t *objspace, const char *fmt, ...), 3, 4);
static const char *obj_info(VALUE obj);
#define PUSH_MARK_FUNC_DATA(v) do { \
struct mark_func_data_struct *prev_mark_func_data = objspace->mark_func_data; \
objspace->mark_func_data = (v);
#define POP_MARK_FUNC_DATA() objspace->mark_func_data = prev_mark_func_data;} while (0)
/*
* 1 - TSC (H/W Time Stamp Counter)
* 2 - getrusage
*/
#ifndef TICK_TYPE
#define TICK_TYPE 1
#endif
#if USE_TICK_T
#if TICK_TYPE == 1
/* the following code is only for internal tuning. */
/* Source code to use RDTSC is quoted and modified from
* http://www.mcs.anl.gov/~kazutomo/rdtsc.html
* written by Kazutomo Yoshii <kazutomo@mcs.anl.gov>
*/
#if defined(__GNUC__) && defined(__i386__)
typedef unsigned long long tick_t;
#define PRItick "llu"
static inline tick_t
tick(void)
{
unsigned long long int x;
__asm__ __volatile__ ("rdtsc" : "=A" (x));
return x;
}
#elif defined(__GNUC__) && defined(__x86_64__)
typedef unsigned long long tick_t;
#define PRItick "llu"
static __inline__ tick_t
tick(void)
{
unsigned long hi, lo;
__asm__ __volatile__ ("rdtsc" : "=a"(lo), "=d"(hi));
return ((unsigned long long)lo)|( ((unsigned long long)hi)<<32);
}
#elif defined(__powerpc64__) && GCC_VERSION_SINCE(4,8,0)
typedef unsigned long long tick_t;
#define PRItick "llu"
static __inline__ tick_t
tick(void)
{
unsigned long long val = __builtin_ppc_get_timebase();
return val;
}
#elif defined(_WIN32) && defined(_MSC_VER)
#include <intrin.h>
typedef unsigned __int64 tick_t;
#define PRItick "llu"
static inline tick_t
tick(void)
{
return __rdtsc();
}
#else /* use clock */
typedef clock_t tick_t;
#define PRItick "llu"
static inline tick_t
tick(void)
{
return clock();
}
#endif /* TSC */
#elif TICK_TYPE == 2
typedef double tick_t;
#define PRItick "4.9f"
static inline tick_t
tick(void)
{
return getrusage_time();
}
#else /* TICK_TYPE */
#error "choose tick type"
#endif /* TICK_TYPE */
#define MEASURE_LINE(expr) do { \
volatile tick_t start_time = tick(); \
volatile tick_t end_time; \
expr; \
end_time = tick(); \
fprintf(stderr, "0\t%"PRItick"\t%s\n", end_time - start_time, #expr); \
} while (0)
#else /* USE_TICK_T */
#define MEASURE_LINE(expr) expr
#endif /* USE_TICK_T */
#define FL_CHECK2(name, x, pred) \
((RGENGC_CHECK_MODE && SPECIAL_CONST_P(x)) ? \
(rb_bug(name": SPECIAL_CONST (%p)", (void *)(x)), 0) : (pred))
#define FL_TEST2(x,f) FL_CHECK2("FL_TEST2", x, FL_TEST_RAW((x),(f)) != 0)
#define FL_SET2(x,f) FL_CHECK2("FL_SET2", x, RBASIC(x)->flags |= (f))
#define FL_UNSET2(x,f) FL_CHECK2("FL_UNSET2", x, RBASIC(x)->flags &= ~(f))
#define RVALUE_MARK_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(obj), (obj))
#define RVALUE_PIN_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_PINNED_BITS(obj), (obj))
#define RVALUE_PAGE_MARKED(page, obj) MARKED_IN_BITMAP((page)->mark_bits, (obj))
#define RVALUE_WB_UNPROTECTED_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), (obj))
#define RVALUE_UNCOLLECTIBLE_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), (obj))
#define RVALUE_MARKING_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), (obj))
#define RVALUE_PAGE_WB_UNPROTECTED(page, obj) MARKED_IN_BITMAP((page)->wb_unprotected_bits, (obj))
#define RVALUE_PAGE_UNCOLLECTIBLE(page, obj) MARKED_IN_BITMAP((page)->uncollectible_bits, (obj))
#define RVALUE_PAGE_MARKING(page, obj) MARKED_IN_BITMAP((page)->marking_bits, (obj))
#define RVALUE_OLD_AGE 3
#define RVALUE_AGE_SHIFT 5 /* FL_PROMOTED0 bit */
static int rgengc_remembered(rb_objspace_t *objspace, VALUE obj);
static int rgengc_remembered_sweep(rb_objspace_t *objspace, VALUE obj);
static int rgengc_remember(rb_objspace_t *objspace, VALUE obj);
static void rgengc_mark_and_rememberset_clear(rb_objspace_t *objspace, rb_heap_t *heap);
static void rgengc_rememberset_mark(rb_objspace_t *objspace, rb_heap_t *heap);
static inline int
RVALUE_FLAGS_AGE(VALUE flags)
{
return (int)((flags & (FL_PROMOTED0 | FL_PROMOTED1)) >> RVALUE_AGE_SHIFT);
}
static int
check_rvalue_consistency_force(const VALUE obj, int terminate)
{
rb_objspace_t *objspace = &rb_objspace;
int err = 0;
if (SPECIAL_CONST_P(obj)) {
fprintf(stderr, "check_rvalue_consistency: %p is a special const.\n", (void *)obj);
err++;
}
else if (!is_pointer_to_heap(objspace, (void *)obj)) {
/* check if it is in tomb_pages */
struct heap_page *page = NULL;
list_for_each(&heap_tomb->pages, page, page_node) {
if (&page->start[0] <= (RVALUE *)obj &&
(RVALUE *)obj < &page->start[page->total_slots]) {
fprintf(stderr, "check_rvalue_consistency: %p is in a tomb_heap (%p).\n",
(void *)obj, (void *)page);
err++;
goto skip;
}
}
fprintf(stderr, "check_rvalue_consistency: %p is not a Ruby object.\n", (void *)obj);
err++;
skip:
;
}
else {
const int wb_unprotected_bit = RVALUE_WB_UNPROTECTED_BITMAP(obj) != 0;
const int uncollectible_bit = RVALUE_UNCOLLECTIBLE_BITMAP(obj) != 0;
const int mark_bit = RVALUE_MARK_BITMAP(obj) != 0;
const int marking_bit = RVALUE_MARKING_BITMAP(obj) != 0, remembered_bit = marking_bit;
const int age = RVALUE_FLAGS_AGE(RBASIC(obj)->flags);
if (GET_HEAP_PAGE(obj)->flags.in_tomb) {
fprintf(stderr, "check_rvalue_consistency: %s is in tomb page.\n", obj_info(obj));
err++;
}
if (BUILTIN_TYPE(obj) == T_NONE) {
fprintf(stderr, "check_rvalue_consistency: %s is T_NONE.\n", obj_info(obj));
err++;
}
if (BUILTIN_TYPE(obj) == T_ZOMBIE) {
fprintf(stderr, "check_rvalue_consistency: %s is T_ZOMBIE.\n", obj_info(obj));
err++;
}
obj_memsize_of((VALUE)obj, FALSE);
/* check generation
*
* OLD == age == 3 && old-bitmap && mark-bit (except incremental marking)
*/
if (age > 0 && wb_unprotected_bit) {
fprintf(stderr, "check_rvalue_consistency: %s is not WB protected, but age is %d > 0.\n", obj_info(obj), age);
err++;
}
if (!is_marking(objspace) && uncollectible_bit && !mark_bit) {
fprintf(stderr, "check_rvalue_consistency: %s is uncollectible, but is not marked while !gc.\n", obj_info(obj));
err++;
}
if (!is_full_marking(objspace)) {
if (uncollectible_bit && age != RVALUE_OLD_AGE && !wb_unprotected_bit) {
fprintf(stderr, "check_rvalue_consistency: %s is uncollectible, but not old (age: %d) and not WB unprotected.\n",
obj_info(obj), age);
err++;
}
if (remembered_bit && age != RVALUE_OLD_AGE) {
fprintf(stderr, "check_rvalue_consistency: %s is remembered, but not old (age: %d).\n",
obj_info(obj), age);
err++;
}
}
/*
* check coloring
*
* marking:false marking:true
* marked:false white *invalid*
* marked:true black grey
*/
if (is_incremental_marking(objspace) && marking_bit) {
if (!is_marking(objspace) && !mark_bit) {
fprintf(stderr, "check_rvalue_consistency: %s is marking, but not marked.\n", obj_info(obj));
err++;
}
}
}
if (err > 0 && terminate) {
rb_bug("check_rvalue_consistency_force: there is %d errors.", err);
}
return err;
}
#if RGENGC_CHECK_MODE == 0
static inline VALUE
check_rvalue_consistency(const VALUE obj)
{
return obj;
}
#else
static VALUE
check_rvalue_consistency(const VALUE obj)
{
check_rvalue_consistency_force(obj, TRUE);
return obj;
}
#endif
static inline int
gc_object_moved_p(rb_objspace_t * objspace, VALUE obj)
{
if (RB_SPECIAL_CONST_P(obj)) {
return FALSE;
}
else {
void *poisoned = asan_poisoned_object_p(obj);
asan_unpoison_object(obj, false);
int ret = BUILTIN_TYPE(obj) == T_MOVED;
/* Re-poison slot if it's not the one we want */
if (poisoned) {
GC_ASSERT(BUILTIN_TYPE(obj) == T_NONE);
asan_poison_object(obj);
}
return ret;
}
}
static inline int
RVALUE_MARKED(VALUE obj)
{
check_rvalue_consistency(obj);
return RVALUE_MARK_BITMAP(obj) != 0;
}
static inline int
RVALUE_PINNED(VALUE obj)
{
check_rvalue_consistency(obj);
return RVALUE_PIN_BITMAP(obj) != 0;
}
static inline int
RVALUE_WB_UNPROTECTED(VALUE obj)
{
check_rvalue_consistency(obj);
return RVALUE_WB_UNPROTECTED_BITMAP(obj) != 0;
}
static inline int
RVALUE_MARKING(VALUE obj)
{
check_rvalue_consistency(obj);
return RVALUE_MARKING_BITMAP(obj) != 0;
}
static inline int
RVALUE_REMEMBERED(VALUE obj)
{
check_rvalue_consistency(obj);
return RVALUE_MARKING_BITMAP(obj) != 0;
}
static inline int
RVALUE_UNCOLLECTIBLE(VALUE obj)
{
check_rvalue_consistency(obj);
return RVALUE_UNCOLLECTIBLE_BITMAP(obj) != 0;
}
static inline int
RVALUE_OLD_P_RAW(VALUE obj)
{
const VALUE promoted = FL_PROMOTED0 | FL_PROMOTED1;
return (RBASIC(obj)->flags & promoted) == promoted;
}
static inline int
RVALUE_OLD_P(VALUE obj)
{
check_rvalue_consistency(obj);
return RVALUE_OLD_P_RAW(obj);
}
#if RGENGC_CHECK_MODE || GC_DEBUG
static inline int
RVALUE_AGE(VALUE obj)
{
check_rvalue_consistency(obj);
return RVALUE_FLAGS_AGE(RBASIC(obj)->flags);
}
#endif
static inline void
RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, struct heap_page *page, VALUE obj)
{
MARK_IN_BITMAP(&page->uncollectible_bits[0], obj);
objspace->rgengc.old_objects++;
rb_transient_heap_promote(obj);
#if RGENGC_PROFILE >= 2
objspace->profile.total_promoted_count++;
objspace->profile.promoted_types[BUILTIN_TYPE(obj)]++;
#endif
}
static inline void
RVALUE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, VALUE obj)
{
RB_DEBUG_COUNTER_INC(obj_promote);
RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(objspace, GET_HEAP_PAGE(obj), obj);
}
static inline VALUE
RVALUE_FLAGS_AGE_SET(VALUE flags, int age)
{
flags &= ~(FL_PROMOTED0 | FL_PROMOTED1);
flags |= (age << RVALUE_AGE_SHIFT);
return flags;
}
/* set age to age+1 */
static inline void
RVALUE_AGE_INC(rb_objspace_t *objspace, VALUE obj)
{
VALUE flags = RBASIC(obj)->flags;
int age = RVALUE_FLAGS_AGE(flags);
if (RGENGC_CHECK_MODE && age == RVALUE_OLD_AGE) {
rb_bug("RVALUE_AGE_INC: can not increment age of OLD object %s.", obj_info(obj));
}
age++;
RBASIC(obj)->flags = RVALUE_FLAGS_AGE_SET(flags, age);
if (age == RVALUE_OLD_AGE) {
RVALUE_OLD_UNCOLLECTIBLE_SET(objspace, obj);
}
check_rvalue_consistency(obj);
}
/* set age to RVALUE_OLD_AGE */
static inline void
RVALUE_AGE_SET_OLD(rb_objspace_t *objspace, VALUE obj)
{
check_rvalue_consistency(obj);
GC_ASSERT(!RVALUE_OLD_P(obj));
RBASIC(obj)->flags = RVALUE_FLAGS_AGE_SET(RBASIC(obj)->flags, RVALUE_OLD_AGE);
RVALUE_OLD_UNCOLLECTIBLE_SET(objspace, obj);
check_rvalue_consistency(obj);
}
/* set age to RVALUE_OLD_AGE - 1 */
static inline void
RVALUE_AGE_SET_CANDIDATE(rb_objspace_t *objspace, VALUE obj)
{
check_rvalue_consistency(obj);
GC_ASSERT(!RVALUE_OLD_P(obj));
RBASIC(obj)->flags = RVALUE_FLAGS_AGE_SET(RBASIC(obj)->flags, RVALUE_OLD_AGE - 1);
check_rvalue_consistency(obj);
}
static inline void
RVALUE_DEMOTE_RAW(rb_objspace_t *objspace, VALUE obj)
{
RBASIC(obj)->flags = RVALUE_FLAGS_AGE_SET(RBASIC(obj)->flags, 0);
CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), obj);
}
static inline void
RVALUE_DEMOTE(rb_objspace_t *objspace, VALUE obj)
{
check_rvalue_consistency(obj);
GC_ASSERT(RVALUE_OLD_P(obj));
if (!is_incremental_marking(objspace) && RVALUE_REMEMBERED(obj)) {
CLEAR_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), obj);
}
RVALUE_DEMOTE_RAW(objspace, obj);
if (RVALUE_MARKED(obj)) {
objspace->rgengc.old_objects--;
}
check_rvalue_consistency(obj);
}
static inline void
RVALUE_AGE_RESET_RAW(VALUE obj)
{
RBASIC(obj)->flags = RVALUE_FLAGS_AGE_SET(RBASIC(obj)->flags, 0);
}
static inline void
RVALUE_AGE_RESET(VALUE obj)
{
check_rvalue_consistency(obj);
GC_ASSERT(!RVALUE_OLD_P(obj));
RVALUE_AGE_RESET_RAW(obj);
check_rvalue_consistency(obj);
}
static inline int
RVALUE_BLACK_P(VALUE obj)
{
return RVALUE_MARKED(obj) && !RVALUE_MARKING(obj);
}
#if 0
static inline int
RVALUE_GREY_P(VALUE obj)
{
return RVALUE_MARKED(obj) && RVALUE_MARKING(obj);
}
#endif
static inline int
RVALUE_WHITE_P(VALUE obj)
{
return RVALUE_MARKED(obj) == FALSE;
}
/*
--------------------------- ObjectSpace -----------------------------
*/
static inline void *
calloc1(size_t n)
{
return calloc(1, n);
}
rb_objspace_t *
rb_objspace_alloc(void)
{
rb_objspace_t *objspace = calloc1(sizeof(rb_objspace_t));
malloc_limit = gc_params.malloc_limit_min;
list_head_init(&objspace->eden_heap.pages);
list_head_init(&objspace->tomb_heap.pages);
dont_gc = TRUE;
return objspace;
}
static void free_stack_chunks(mark_stack_t *);
static void heap_page_free(rb_objspace_t *objspace, struct heap_page *page);
void
rb_objspace_free(rb_objspace_t *objspace)
{
if (is_lazy_sweeping(heap_eden))
rb_bug("lazy sweeping underway when freeing object space");
if (objspace->profile.records) {
free(objspace->profile.records);
objspace->profile.records = 0;
}
if (global_list) {
struct gc_list *list, *next;
for (list = global_list; list; list = next) {
next = list->next;
xfree(list);
}
}
if (heap_pages_sorted) {
size_t i;
for (i = 0; i < heap_allocated_pages; ++i) {
heap_page_free(objspace, heap_pages_sorted[i]);
}
free(heap_pages_sorted);
heap_allocated_pages = 0;
heap_pages_sorted_length = 0;
heap_pages_lomem = 0;
heap_pages_himem = 0;
objspace->eden_heap.total_pages = 0;
objspace->eden_heap.total_slots = 0;
}
st_free_table(objspace->id_to_obj_tbl);
st_free_table(objspace->obj_to_id_tbl);
free_stack_chunks(&objspace->mark_stack);
free(objspace);
}
static void
heap_pages_expand_sorted_to(rb_objspace_t *objspace, size_t next_length)
{
struct heap_page **sorted;
size_t size = size_mul_or_raise(next_length, sizeof(struct heap_page *), rb_eRuntimeError);
gc_report(3, objspace, "heap_pages_expand_sorted: next_length: %d, size: %d\n", (int)next_length, (int)size);
if (heap_pages_sorted_length > 0) {
sorted = (struct heap_page **)realloc(heap_pages_sorted, size);
if (sorted) heap_pages_sorted = sorted;
}
else {
sorted = heap_pages_sorted = (struct heap_page **)malloc(size);
}
if (sorted == 0) {
rb_memerror();
}
heap_pages_sorted_length = next_length;
}
static void
heap_pages_expand_sorted(rb_objspace_t *objspace)
{
/* usually heap_allocatable_pages + heap_eden->total_pages == heap_pages_sorted_length
* because heap_allocatable_pages contains heap_tomb->total_pages (recycle heap_tomb pages).
* however, if there are pages which do not have empty slots, then try to create new pages
* so that the additional allocatable_pages counts (heap_tomb->total_pages) are added.
*/
size_t next_length = heap_allocatable_pages;
next_length += heap_eden->total_pages;
next_length += heap_tomb->total_pages;
if (next_length > heap_pages_sorted_length) {
heap_pages_expand_sorted_to(objspace, next_length);
}
GC_ASSERT(heap_allocatable_pages + heap_eden->total_pages <= heap_pages_sorted_length);
GC_ASSERT(heap_allocated_pages <= heap_pages_sorted_length);
}
static void
heap_allocatable_pages_set(rb_objspace_t *objspace, size_t s)
{
heap_allocatable_pages = s;
heap_pages_expand_sorted(objspace);
}
static inline void
heap_page_add_freeobj(rb_objspace_t *objspace, struct heap_page *page, VALUE obj)
{
RVALUE *p = (RVALUE *)obj;
asan_unpoison_memory_region(&page->freelist, sizeof(RVALUE*), false);
p->as.free.flags = 0;
p->as.free.next = page->freelist;
page->freelist = p;
asan_poison_memory_region(&page->freelist, sizeof(RVALUE*));
if (RGENGC_CHECK_MODE &&
/* obj should belong to page */
!(&page->start[0] <= (RVALUE *)obj &&
(RVALUE *)obj < &page->start[page->total_slots] &&
obj % sizeof(RVALUE) == 0)) {
rb_bug("heap_page_add_freeobj: %p is not rvalue.", (void *)p);
}
asan_poison_object(obj);
gc_report(3, objspace, "heap_page_add_freeobj: add %p to freelist\n", (void *)obj);
}
static inline void
heap_add_freepage(rb_heap_t *heap, struct heap_page *page)
{
asan_unpoison_memory_region(&page->freelist, sizeof(RVALUE*), false);
GC_ASSERT(page->free_slots != 0);
if (page->freelist) {
page->free_next = heap->free_pages;
heap->free_pages = page;
}
asan_poison_memory_region(&page->freelist, sizeof(RVALUE*));
}
#if GC_ENABLE_INCREMENTAL_MARK
static inline int
heap_add_poolpage(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
{
asan_unpoison_memory_region(&page->freelist, sizeof(RVALUE*), false);
if (page->freelist) {
page->free_next = heap->pooled_pages;
heap->pooled_pages = page;
objspace->rincgc.pooled_slots += page->free_slots;
asan_poison_memory_region(&page->freelist, sizeof(RVALUE*));
return TRUE;
}
else {
asan_poison_memory_region(&page->freelist, sizeof(RVALUE*));
return FALSE;
}
}
#endif
static void
heap_unlink_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
{
list_del(&page->page_node);
heap->total_pages--;
heap->total_slots -= page->total_slots;
}
static void rb_aligned_free(void *ptr);
static void
heap_page_free(rb_objspace_t *objspace, struct heap_page *page)
{
heap_allocated_pages--;
objspace->profile.total_freed_pages++;
rb_aligned_free(GET_PAGE_BODY(page->start));
free(page);
}
static void
heap_pages_free_unused_pages(rb_objspace_t *objspace)
{
size_t i, j;
if (!list_empty(&heap_tomb->pages)) {
for (i = j = 1; j < heap_allocated_pages; i++) {
struct heap_page *page = heap_pages_sorted[i];
if (page->flags.in_tomb && page->free_slots == page->total_slots) {
heap_unlink_page(objspace, heap_tomb, page);
heap_page_free(objspace, page);
}
else {
if (i != j) {
heap_pages_sorted[j] = page;
}
j++;
}
}
GC_ASSERT(j == heap_allocated_pages);
}
}
static struct heap_page *
heap_page_allocate(rb_objspace_t *objspace)
{
RVALUE *start, *end, *p;
struct heap_page *page;
struct heap_page_body *page_body = 0;
size_t hi, lo, mid;
int limit = HEAP_PAGE_OBJ_LIMIT;
/* assign heap_page body (contains heap_page_header and RVALUEs) */
page_body = (struct heap_page_body *)rb_aligned_malloc(HEAP_PAGE_ALIGN, HEAP_PAGE_SIZE);
if (page_body == 0) {
rb_memerror();
}
/* assign heap_page entry */
page = calloc1(sizeof(struct heap_page));
if (page == 0) {
rb_aligned_free(page_body);
rb_memerror();
}
/* adjust obj_limit (object number available in this page) */
start = (RVALUE*)((VALUE)page_body + sizeof(struct heap_page_header));
if ((VALUE)start % sizeof(RVALUE) != 0) {
int delta = (int)(sizeof(RVALUE) - ((VALUE)start % sizeof(RVALUE)));
start = (RVALUE*)((VALUE)start + delta);
limit = (HEAP_PAGE_SIZE - (int)((VALUE)start - (VALUE)page_body))/(int)sizeof(RVALUE);
}
end = start + limit;
/* setup heap_pages_sorted */
lo = 0;
hi = heap_allocated_pages;
while (lo < hi) {
struct heap_page *mid_page;
mid = (lo + hi) / 2;
mid_page = heap_pages_sorted[mid];
if (mid_page->start < start) {
lo = mid + 1;
}
else if (mid_page->start > start) {
hi = mid;
}
else {
rb_bug("same heap page is allocated: %p at %"PRIuVALUE, (void *)page_body, (VALUE)mid);
}
}
if (hi < heap_allocated_pages) {
MEMMOVE(&heap_pages_sorted[hi+1], &heap_pages_sorted[hi], struct heap_page_header*, heap_allocated_pages - hi);
}
heap_pages_sorted[hi] = page;
heap_allocated_pages++;
GC_ASSERT(heap_eden->total_pages + heap_allocatable_pages <= heap_pages_sorted_length);
GC_ASSERT(heap_eden->total_pages + heap_tomb->total_pages == heap_allocated_pages - 1);
GC_ASSERT(heap_allocated_pages <= heap_pages_sorted_length);
objspace->profile.total_allocated_pages++;
if (heap_allocated_pages > heap_pages_sorted_length) {
rb_bug("heap_page_allocate: allocated(%"PRIdSIZE") > sorted(%"PRIdSIZE")",
heap_allocated_pages, heap_pages_sorted_length);
}
if (heap_pages_lomem == 0 || heap_pages_lomem > start) heap_pages_lomem = start;
if (heap_pages_himem < end) heap_pages_himem = end;
page->start = start;
page->total_slots = limit;
page_body->header.page = page;
for (p = start; p != end; p++) {
gc_report(3, objspace, "assign_heap_page: %p is added to freelist\n", (void *)p);
heap_page_add_freeobj(objspace, page, (VALUE)p);
}
page->free_slots = limit;
asan_poison_memory_region(&page->freelist, sizeof(RVALUE*));
return page;
}
static struct heap_page *
heap_page_resurrect(rb_objspace_t *objspace)
{
struct heap_page *page = 0, *next;
list_for_each_safe(&heap_tomb->pages, page, next, page_node) {
asan_unpoison_memory_region(&page->freelist, sizeof(RVALUE*), false);
if (page->freelist != NULL) {
heap_unlink_page(objspace, heap_tomb, page);
asan_poison_memory_region(&page->freelist, sizeof(RVALUE*));
return page;
}
}
return NULL;
}
static struct heap_page *
heap_page_create(rb_objspace_t *objspace)
{
struct heap_page *page;
const char *method = "recycle";
heap_allocatable_pages--;
page = heap_page_resurrect(objspace);
if (page == NULL) {
page = heap_page_allocate(objspace);
method = "allocate";
}
if (0) fprintf(stderr, "heap_page_create: %s - %p, heap_allocated_pages: %d, heap_allocated_pages: %d, tomb->total_pages: %d\n",
method, (void *)page, (int)heap_pages_sorted_length, (int)heap_allocated_pages, (int)heap_tomb->total_pages);
return page;
}
static void
heap_add_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
{
page->flags.in_tomb = (heap == heap_tomb);
list_add(&heap->pages, &page->page_node);
heap->total_pages++;
heap->total_slots += page->total_slots;
}
static void
heap_assign_page(rb_objspace_t *objspace, rb_heap_t *heap)
{
struct heap_page *page = heap_page_create(objspace);
heap_add_page(objspace, heap, page);
heap_add_freepage(heap, page);
}
static void
heap_add_pages(rb_objspace_t *objspace, rb_heap_t *heap, size_t add)
{
size_t i;
heap_allocatable_pages_set(objspace, add);
for (i = 0; i < add; i++) {
heap_assign_page(objspace, heap);
}
GC_ASSERT(heap_allocatable_pages == 0);
}
static size_t
heap_extend_pages(rb_objspace_t *objspace, size_t free_slots, size_t total_slots)
{
double goal_ratio = gc_params.heap_free_slots_goal_ratio;
size_t used = heap_allocated_pages + heap_allocatable_pages;
size_t next_used;
if (goal_ratio == 0.0) {
next_used = (size_t)(used * gc_params.growth_factor);
}
else {
/* Find `f' where free_slots = f * total_slots * goal_ratio
* => f = (total_slots - free_slots) / ((1 - goal_ratio) * total_slots)
*/
double f = (double)(total_slots - free_slots) / ((1 - goal_ratio) * total_slots);
if (f > gc_params.growth_factor) f = gc_params.growth_factor;
if (f < 1.0) f = 1.1;
next_used = (size_t)(f * used);
if (0) {
fprintf(stderr,
"free_slots(%8"PRIuSIZE")/total_slots(%8"PRIuSIZE")=%1.2f,"
" G(%1.2f), f(%1.2f),"
" used(%8"PRIuSIZE") => next_used(%8"PRIuSIZE")\n",
free_slots, total_slots, free_slots/(double)total_slots,
goal_ratio, f, used, next_used);
}
}
if (gc_params.growth_max_slots > 0) {
size_t max_used = (size_t)(used + gc_params.growth_max_slots/HEAP_PAGE_OBJ_LIMIT);
if (next_used > max_used) next_used = max_used;
}
return next_used - used;
}
static void
heap_set_increment(rb_objspace_t *objspace, size_t additional_pages)
{
size_t used = heap_eden->total_pages;
size_t next_used_limit = used + additional_pages;
if (next_used_limit == heap_allocated_pages) next_used_limit++;
heap_allocatable_pages_set(objspace, next_used_limit - used);
gc_report(1, objspace, "heap_set_increment: heap_allocatable_pages is %d\n", (int)heap_allocatable_pages);
}
static int
heap_increment(rb_objspace_t *objspace, rb_heap_t *heap)
{
if (heap_allocatable_pages > 0) {
gc_report(1, objspace, "heap_increment: heap_pages_sorted_length: %d, heap_pages_inc: %d, heap->total_pages: %d\n",
(int)heap_pages_sorted_length, (int)heap_allocatable_pages, (int)heap->total_pages);
GC_ASSERT(heap_allocatable_pages + heap_eden->total_pages <= heap_pages_sorted_length);
GC_ASSERT(heap_allocated_pages <= heap_pages_sorted_length);
heap_assign_page(objspace, heap);
return TRUE;
}
return FALSE;
}
static void
heap_prepare(rb_objspace_t *objspace, rb_heap_t *heap)
{
GC_ASSERT(heap->free_pages == NULL);
if (is_lazy_sweeping(heap)) {
gc_sweep_continue(objspace, heap);
}
else if (is_incremental_marking(objspace)) {
gc_marks_continue(objspace, heap);
}
if (heap->free_pages == NULL &&
(will_be_incremental_marking(objspace) || heap_increment(objspace, heap) == FALSE) &&
gc_start(objspace, GPR_FLAG_NEWOBJ) == FALSE) {
rb_memerror();
}
}
static RVALUE *
heap_get_freeobj_from_next_freepage(rb_objspace_t *objspace, rb_heap_t *heap)
{
struct heap_page *page;
RVALUE *p;
while (heap->free_pages == NULL) {
heap_prepare(objspace, heap);
}
page = heap->free_pages;
heap->free_pages = page->free_next;
heap->using_page = page;
GC_ASSERT(page->free_slots != 0);
asan_unpoison_memory_region(&page->freelist, sizeof(RVALUE*), false);
p = page->freelist;
page->freelist = NULL;
asan_poison_memory_region(&page->freelist, sizeof(RVALUE*));
page->free_slots = 0;
asan_unpoison_object((VALUE)p, true);
return p;
}
static inline VALUE
heap_get_freeobj_head(rb_objspace_t *objspace, rb_heap_t *heap)
{
RVALUE *p = heap->freelist;
if (LIKELY(p != NULL)) {
heap->freelist = p->as.free.next;
}
asan_unpoison_object((VALUE)p, true);
return (VALUE)p;
}
static inline VALUE
heap_get_freeobj(rb_objspace_t *objspace, rb_heap_t *heap)
{
RVALUE *p = heap->freelist;
while (1) {
if (LIKELY(p != NULL)) {
asan_unpoison_object((VALUE)p, true);
heap->freelist = p->as.free.next;
return (VALUE)p;
}
else {
p = heap_get_freeobj_from_next_freepage(objspace, heap);
}
}
}
void
rb_objspace_set_event_hook(const rb_event_flag_t event)
{
rb_objspace_t *objspace = &rb_objspace;
objspace->hook_events = event & RUBY_INTERNAL_EVENT_OBJSPACE_MASK;
objspace->flags.has_hook = (objspace->hook_events != 0);
}
static void
gc_event_hook_body(rb_execution_context_t *ec, rb_objspace_t *objspace, const rb_event_flag_t event, VALUE data)
{
const VALUE *pc = ec->cfp->pc;
if (pc && VM_FRAME_RUBYFRAME_P(ec->cfp)) {
/* increment PC because source line is calculated with PC-1 */
ec->cfp->pc++;
}
EXEC_EVENT_HOOK(ec, event, ec->cfp->self, 0, 0, 0, data);
ec->cfp->pc = pc;
}
#define gc_event_hook_available_p(objspace) ((objspace)->flags.has_hook)
#define gc_event_hook_needed_p(objspace, event) ((objspace)->hook_events & (event))
#define gc_event_hook(objspace, event, data) do { \
if (UNLIKELY(gc_event_hook_needed_p(objspace, event))) { \
gc_event_hook_body(GET_EC(), (objspace), (event), (data)); \
} \
} while (0)
static inline VALUE
newobj_init(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, int wb_protected, rb_objspace_t *objspace, VALUE obj)
{
#if !__has_feature(memory_sanitizer)
GC_ASSERT(BUILTIN_TYPE(obj) == T_NONE);
GC_ASSERT((flags & FL_WB_PROTECTED) == 0);
#endif
/* OBJSETUP */
struct RVALUE buf = {
.as = {
.values = {
.basic = {
.flags = flags,
.klass = klass,
},
.v1 = v1,
.v2 = v2,
.v3 = v3,
},
},
};
MEMCPY(RANY(obj), &buf, RVALUE, 1);
#if RGENGC_CHECK_MODE
GC_ASSERT(RVALUE_MARKED(obj) == FALSE);
GC_ASSERT(RVALUE_MARKING(obj) == FALSE);
GC_ASSERT(RVALUE_OLD_P(obj) == FALSE);
GC_ASSERT(RVALUE_WB_UNPROTECTED(obj) == FALSE);
if (flags & FL_PROMOTED1) {
if (RVALUE_AGE(obj) != 2) rb_bug("newobj: %s of age (%d) != 2.", obj_info(obj), RVALUE_AGE(obj));
}
else {
if (RVALUE_AGE(obj) > 0) rb_bug("newobj: %s of age (%d) > 0.", obj_info(obj), RVALUE_AGE(obj));
}
if (rgengc_remembered(objspace, (VALUE)obj)) rb_bug("newobj: %s is remembered.", obj_info(obj));
#endif
if (UNLIKELY(wb_protected == FALSE)) {
MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
}
#if RGENGC_PROFILE
if (wb_protected) {
objspace->profile.total_generated_normal_object_count++;
#if RGENGC_PROFILE >= 2
objspace->profile.generated_normal_object_count_types[BUILTIN_TYPE(obj)]++;
#endif
}
else {
objspace->profile.total_generated_shady_object_count++;
#if RGENGC_PROFILE >= 2
objspace->profile.generated_shady_object_count_types[BUILTIN_TYPE(obj)]++;
#endif
}
#endif
#if GC_DEBUG
RANY(obj)->file = rb_source_location_cstr(&RANY(obj)->line);
GC_ASSERT(!SPECIAL_CONST_P(obj)); /* check alignment */
#endif
objspace->total_allocated_objects++;
gc_report(5, objspace, "newobj: %s\n", obj_info(obj));
#if RGENGC_OLD_NEWOBJ_CHECK > 0
{
static int newobj_cnt = RGENGC_OLD_NEWOBJ_CHECK;
if (!is_incremental_marking(objspace) &&
flags & FL_WB_PROTECTED && /* do not promote WB unprotected objects */
! RB_TYPE_P(obj, T_ARRAY)) { /* array.c assumes that allocated objects are new */
if (--newobj_cnt == 0) {
newobj_cnt = RGENGC_OLD_NEWOBJ_CHECK;
gc_mark_set(objspace, obj);
RVALUE_AGE_SET_OLD(objspace, obj);
rb_gc_writebarrier_remember(obj);
}
}
}
#endif
check_rvalue_consistency(obj);
return obj;
}
static inline VALUE
newobj_slowpath(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, rb_objspace_t *objspace, int wb_protected)
{
VALUE obj;
if (UNLIKELY(during_gc || ruby_gc_stressful)) {
if (during_gc) {
dont_gc = 1;
during_gc = 0;
rb_bug("object allocation during garbage collection phase");
}
if (ruby_gc_stressful) {
if (!garbage_collect(objspace, GPR_FLAG_NEWOBJ)) {
rb_memerror();
}
}
}
obj = heap_get_freeobj(objspace, heap_eden);
newobj_init(klass, flags, v1, v2, v3, wb_protected, objspace, obj);
gc_event_hook(objspace, RUBY_INTERNAL_EVENT_NEWOBJ, obj);
return obj;
}
NOINLINE(static VALUE newobj_slowpath_wb_protected(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, rb_objspace_t *objspace));
NOINLINE(static VALUE newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, rb_objspace_t *objspace));
static VALUE
newobj_slowpath_wb_protected(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, rb_objspace_t *objspace)
{
return newobj_slowpath(klass, flags, v1, v2, v3, objspace, TRUE);
}
static VALUE
newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, rb_objspace_t *objspace)
{
return newobj_slowpath(klass, flags, v1, v2, v3, objspace, FALSE);
}
static inline VALUE
newobj_of(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, int wb_protected)
{
rb_objspace_t *objspace = &rb_objspace;
VALUE obj;
RB_DEBUG_COUNTER_INC(obj_newobj);
(void)RB_DEBUG_COUNTER_INC_IF(obj_newobj_wb_unprotected, !wb_protected);
#if GC_DEBUG_STRESS_TO_CLASS
if (UNLIKELY(stress_to_class)) {
long i, cnt = RARRAY_LEN(stress_to_class);
for (i = 0; i < cnt; ++i) {
if (klass == RARRAY_AREF(stress_to_class, i)) rb_memerror();
}
}
#endif
if (!(during_gc ||
ruby_gc_stressful ||
gc_event_hook_available_p(objspace)) &&
(obj = heap_get_freeobj_head(objspace, heap_eden)) != Qfalse) {
return newobj_init(klass, flags, v1, v2, v3, wb_protected, objspace, obj);
}
else {
RB_DEBUG_COUNTER_INC(obj_newobj_slowpath);
return wb_protected ?
newobj_slowpath_wb_protected(klass, flags, v1, v2, v3, objspace) :
newobj_slowpath_wb_unprotected(klass, flags, v1, v2, v3, objspace);
}
}
VALUE
rb_wb_unprotected_newobj_of(VALUE klass, VALUE flags)
{
GC_ASSERT((flags & FL_WB_PROTECTED) == 0);
return newobj_of(klass, flags, 0, 0, 0, FALSE);
}
VALUE
rb_wb_protected_newobj_of(VALUE klass, VALUE flags)
{
GC_ASSERT((flags & FL_WB_PROTECTED) == 0);
return newobj_of(klass, flags, 0, 0, 0, TRUE);
}
/* for compatibility */
VALUE
rb_newobj(void)
{
return newobj_of(0, T_NONE, 0, 0, 0, FALSE);
}
VALUE
rb_newobj_of(VALUE klass, VALUE flags)
{
return newobj_of(klass, flags & ~FL_WB_PROTECTED, 0, 0, 0, flags & FL_WB_PROTECTED);
}
#define UNEXPECTED_NODE(func) \
rb_bug(#func"(): GC does not handle T_NODE 0x%x(%p) 0x%"PRIxVALUE, \
BUILTIN_TYPE(obj), (void*)(obj), RBASIC(obj)->flags)
#undef rb_imemo_new
VALUE
rb_imemo_new(enum imemo_type type, VALUE v1, VALUE v2, VALUE v3, VALUE v0)
{
VALUE flags = T_IMEMO | (type << FL_USHIFT);
return newobj_of(v0, flags, v1, v2, v3, TRUE);
}
static VALUE
rb_imemo_tmpbuf_new(VALUE v1, VALUE v2, VALUE v3, VALUE v0)
{
VALUE flags = T_IMEMO | (imemo_tmpbuf << FL_USHIFT);
return newobj_of(v0, flags, v1, v2, v3, FALSE);
}
static VALUE
rb_imemo_tmpbuf_auto_free_maybe_mark_buffer(void *buf, size_t cnt)
{
return rb_imemo_tmpbuf_new((VALUE)buf, 0, (VALUE)cnt, 0);
}
rb_imemo_tmpbuf_t *
rb_imemo_tmpbuf_parser_heap(void *buf, rb_imemo_tmpbuf_t *old_heap, size_t cnt)
{
return (rb_imemo_tmpbuf_t *)rb_imemo_tmpbuf_new((VALUE)buf, (VALUE)old_heap, (VALUE)cnt, 0);
}
static size_t
imemo_memsize(VALUE obj)
{
size_t size = 0;
switch (imemo_type(obj)) {
case imemo_ment:
size += sizeof(RANY(obj)->as.imemo.ment.def);
break;
case imemo_iseq:
size += rb_iseq_memsize((rb_iseq_t *)obj);
break;
case imemo_env:
size += RANY(obj)->as.imemo.env.env_size * sizeof(VALUE);
break;
case imemo_tmpbuf:
size += RANY(obj)->as.imemo.alloc.cnt * sizeof(VALUE);
break;
case imemo_ast:
size += rb_ast_memsize(&RANY(obj)->as.imemo.ast);
break;
case imemo_cref:
case imemo_svar:
case imemo_throw_data:
case imemo_ifunc:
case imemo_memo:
case imemo_parser_strterm:
break;
default:
/* unreachable */
break;
}
return size;
}
#if IMEMO_DEBUG
VALUE
rb_imemo_new_debug(enum imemo_type type, VALUE v1, VALUE v2, VALUE v3, VALUE v0, const char *file, int line)
{
VALUE memo = rb_imemo_new(type, v1, v2, v3, v0);
fprintf(stderr, "memo %p (type: %d) @ %s:%d\n", (void *)memo, imemo_type(memo), file, line);
return memo;
}
#endif
VALUE
rb_data_object_wrap(VALUE klass, void *datap, RUBY_DATA_FUNC dmark, RUBY_DATA_FUNC dfree)
{
RUBY_ASSERT_ALWAYS(dfree != (RUBY_DATA_FUNC)1);
if (klass) Check_Type(klass, T_CLASS);
return newobj_of(klass, T_DATA, (VALUE)dmark, (VALUE)dfree, (VALUE)datap, FALSE);
}
VALUE
rb_data_object_zalloc(VALUE klass, size_t size, RUBY_DATA_FUNC dmark, RUBY_DATA_FUNC dfree)
{
VALUE obj = rb_data_object_wrap(klass, 0, dmark, dfree);
DATA_PTR(obj) = xcalloc(1, size);
return obj;
}
VALUE
rb_data_typed_object_wrap(VALUE klass, void *datap, const rb_data_type_t *type)
{
RUBY_ASSERT_ALWAYS(type);
if (klass) Check_Type(klass, T_CLASS);
return newobj_of(klass, T_DATA, (VALUE)type, (VALUE)1, (VALUE)datap, type->flags & RUBY_FL_WB_PROTECTED);
}
VALUE
rb_data_typed_object_zalloc(VALUE klass, size_t size, const rb_data_type_t *type)
{
VALUE obj = rb_data_typed_object_wrap(klass, 0, type);
DATA_PTR(obj) = xcalloc(1, size);
return obj;
}
size_t
rb_objspace_data_type_memsize(VALUE obj)
{
if (RTYPEDDATA_P(obj)) {
const rb_data_type_t *type = RTYPEDDATA_TYPE(obj);
const void *ptr = RTYPEDDATA_DATA(obj);
if (ptr && type->function.dsize) {
return type->function.dsize(ptr);
}
}
return 0;
}
const char *
rb_objspace_data_type_name(VALUE obj)
{
if (RTYPEDDATA_P(obj)) {
return RTYPEDDATA_TYPE(obj)->wrap_struct_name;
}
else {
return 0;
}
}
PUREFUNC(static inline int is_pointer_to_heap(rb_objspace_t *objspace, void *ptr);)
static inline int
is_pointer_to_heap(rb_objspace_t *objspace, void *ptr)
{
register RVALUE *p = RANY(ptr);
register struct heap_page *page;
register size_t hi, lo, mid;
RB_DEBUG_COUNTER_INC(gc_isptr_trial);
if (p < heap_pages_lomem || p > heap_pages_himem) return FALSE;
RB_DEBUG_COUNTER_INC(gc_isptr_range);
if ((VALUE)p % sizeof(RVALUE) != 0) return FALSE;
RB_DEBUG_COUNTER_INC(gc_isptr_align);
/* check if p looks like a pointer using bsearch*/
lo = 0;
hi = heap_allocated_pages;
while (lo < hi) {
mid = (lo + hi) / 2;
page = heap_pages_sorted[mid];
if (page->start <= p) {
if (p < page->start + page->total_slots) {
RB_DEBUG_COUNTER_INC(gc_isptr_maybe);
if (page->flags.in_tomb) {
return FALSE;
}
else {
return TRUE;
}
}
lo = mid + 1;
}
else {
hi = mid;
}
}
return FALSE;
}
static enum rb_id_table_iterator_result
free_const_entry_i(VALUE value, void *data)
{
rb_const_entry_t *ce = (rb_const_entry_t *)value;
xfree(ce);
return ID_TABLE_CONTINUE;
}
void
rb_free_const_table(struct rb_id_table *tbl)
{
rb_id_table_foreach_values(tbl, free_const_entry_i, 0);
rb_id_table_free(tbl);
}
// alive: if false, target pointers can be freed already.
// To check it, we need objspace parameter.
static void
vm_ccs_free(struct rb_class_cc_entries *ccs, int alive, rb_objspace_t *objspace, VALUE klass)
{
if (ccs->entries) {
for (int i=0; i<ccs->len; i++) {
const struct rb_callcache *cc = ccs->entries[i].cc;
if (!alive) {
// ccs can be free'ed.
if (is_pointer_to_heap(objspace, (void *)cc) &&
IMEMO_TYPE_P(cc, imemo_callcache) &&
cc->klass == klass) {
// OK. maybe target cc.
}
else {
continue;
}
}
vm_cc_invalidate(cc);
}
ruby_xfree(ccs->entries);
}
ruby_xfree(ccs);
}
void
rb_vm_ccs_free(struct rb_class_cc_entries *ccs)
{
RB_DEBUG_COUNTER_INC(ccs_free);
vm_ccs_free(ccs, TRUE, NULL, Qundef);
}
struct cc_tbl_i_data {
rb_objspace_t *objspace;
VALUE klass;
bool alive;
};
static enum rb_id_table_iterator_result
cc_table_mark_i(ID id, VALUE ccs_ptr, void *data_ptr)
{
struct cc_tbl_i_data *data = data_ptr;
struct rb_class_cc_entries *ccs = (struct rb_class_cc_entries *)ccs_ptr;
VM_ASSERT(vm_ccs_p(ccs));
VM_ASSERT(id == ccs->cme->called_id);
if (METHOD_ENTRY_INVALIDATED(ccs->cme)) {
rb_vm_ccs_free(ccs);
return ID_TABLE_DELETE;
}
else {
gc_mark(data->objspace, (VALUE)ccs->cme);
for (int i=0; i<ccs->len; i++) {
VM_ASSERT(data->klass == ccs->entries[i].cc->klass);
VM_ASSERT(ccs->cme == vm_cc_cme(ccs->entries[i].cc));
gc_mark(data->objspace, (VALUE)ccs->entries[i].ci);
gc_mark(data->objspace, (VALUE)ccs->entries[i].cc);
}
return ID_TABLE_CONTINUE;
}
}
static void
cc_table_mark(rb_objspace_t *objspace, VALUE klass)
{
struct rb_id_table *cc_tbl = RCLASS_CC_TBL(klass);
if (cc_tbl) {
struct cc_tbl_i_data data = {
.objspace = objspace,
.klass = klass,
};
rb_id_table_foreach(cc_tbl, cc_table_mark_i, &data);
}
}
static enum rb_id_table_iterator_result
cc_table_free_i(ID id, VALUE ccs_ptr, void *data_ptr)
{
struct cc_tbl_i_data *data = data_ptr;
struct rb_class_cc_entries *ccs = (struct rb_class_cc_entries *)ccs_ptr;
VM_ASSERT(vm_ccs_p(ccs));
vm_ccs_free(ccs, data->alive, data->objspace, data->klass);
return ID_TABLE_CONTINUE;
}
static void
cc_table_free(rb_objspace_t *objspace, VALUE klass, bool alive)
{
struct rb_id_table *cc_tbl = RCLASS_CC_TBL(klass);
if (cc_tbl) {
struct cc_tbl_i_data data = {
.objspace = objspace,
.klass = klass,
.alive = alive,
};
rb_id_table_foreach(cc_tbl, cc_table_free_i, &data);
rb_id_table_free(cc_tbl);
}
}
void
rb_cc_table_free(VALUE klass)
{
cc_table_free(&rb_objspace, klass, TRUE);
}
static inline void
make_zombie(rb_objspace_t *objspace, VALUE obj, void (*dfree)(void *), void *data)
{
struct RZombie *zombie = RZOMBIE(obj);
zombie->basic.flags = T_ZOMBIE | (zombie->basic.flags & FL_SEEN_OBJ_ID);
zombie->dfree = dfree;
zombie->data = data;
zombie->next = heap_pages_deferred_final;
heap_pages_deferred_final = (VALUE)zombie;
}
static inline void
make_io_zombie(rb_objspace_t *objspace, VALUE obj)
{
rb_io_t *fptr = RANY(obj)->as.file.fptr;
make_zombie(objspace, obj, (void (*)(void*))rb_io_fptr_finalize, fptr);
}
static void
obj_free_object_id(rb_objspace_t *objspace, VALUE obj)
{
st_data_t o = (st_data_t)obj, id;
GC_ASSERT(FL_TEST(obj, FL_SEEN_OBJ_ID));
FL_UNSET(obj, FL_SEEN_OBJ_ID);
if (st_delete(objspace->obj_to_id_tbl, &o, &id)) {
GC_ASSERT(id);
st_delete(objspace->id_to_obj_tbl, &id, NULL);
}
else {
rb_bug("Object ID seen, but not in mapping table: %s\n", obj_info(obj));
}
}
static int
obj_free(rb_objspace_t *objspace, VALUE obj)
{
RB_DEBUG_COUNTER_INC(obj_free);
gc_event_hook(objspace, RUBY_INTERNAL_EVENT_FREEOBJ, obj);
switch (BUILTIN_TYPE(obj)) {
case T_NIL:
case T_FIXNUM:
case T_TRUE:
case T_FALSE:
rb_bug("obj_free() called for broken object");
break;
default:
break;
}
if (FL_TEST(obj, FL_EXIVAR)) {
rb_free_generic_ivar((VALUE)obj);
FL_UNSET(obj, FL_EXIVAR);
}
if (FL_TEST(obj, FL_SEEN_OBJ_ID) && !FL_TEST(obj, FL_FINALIZE)) {
obj_free_object_id(objspace, obj);
}
if (RVALUE_WB_UNPROTECTED(obj)) CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
#if RGENGC_CHECK_MODE
#define CHECK(x) if (x(obj) != FALSE) rb_bug("obj_free: " #x "(%s) != FALSE", obj_info(obj))
CHECK(RVALUE_WB_UNPROTECTED);
CHECK(RVALUE_MARKED);
CHECK(RVALUE_MARKING);
CHECK(RVALUE_UNCOLLECTIBLE);
#undef CHECK
#endif
switch (BUILTIN_TYPE(obj)) {
case T_OBJECT:
if ((RANY(obj)->as.basic.flags & ROBJECT_EMBED) ||
RANY(obj)->as.object.as.heap.ivptr == NULL) {
RB_DEBUG_COUNTER_INC(obj_obj_embed);
}
else if (ROBJ_TRANSIENT_P(obj)) {
RB_DEBUG_COUNTER_INC(obj_obj_transient);
}
else {
xfree(RANY(obj)->as.object.as.heap.ivptr);
RB_DEBUG_COUNTER_INC(obj_obj_ptr);
}
break;
case T_MODULE:
case T_CLASS:
mjit_remove_class_serial(RCLASS_SERIAL(obj));
rb_id_table_free(RCLASS_M_TBL(obj));
cc_table_free(objspace, obj, FALSE);
if (RCLASS_IV_TBL(obj)) {
st_free_table(RCLASS_IV_TBL(obj));
}
if (RCLASS_CONST_TBL(obj)) {
rb_free_const_table(RCLASS_CONST_TBL(obj));
}
if (RCLASS_IV_INDEX_TBL(obj)) {
st_free_table(RCLASS_IV_INDEX_TBL(obj));
}
if (RCLASS_EXT(obj)->subclasses) {
if (BUILTIN_TYPE(obj) == T_MODULE) {
rb_class_detach_module_subclasses(obj);
}
else {
rb_class_detach_subclasses(obj);
}
RCLASS_EXT(obj)->subclasses = NULL;
}
rb_class_remove_from_module_subclasses(obj);
rb_class_remove_from_super_subclasses(obj);
if (RANY(obj)->as.klass.ptr)
xfree(RANY(obj)->as.klass.ptr);
RANY(obj)->as.klass.ptr = NULL;
(void)RB_DEBUG_COUNTER_INC_IF(obj_module_ptr, BUILTIN_TYPE(obj) == T_MODULE);
(void)RB_DEBUG_COUNTER_INC_IF(obj_class_ptr, BUILTIN_TYPE(obj) == T_CLASS);
break;
case T_STRING:
rb_str_free(obj);
break;
case T_ARRAY:
rb_ary_free(obj);
break;
case T_HASH:
#if USE_DEBUG_COUNTER
switch (RHASH_SIZE(obj)) {
case 0:
RB_DEBUG_COUNTER_INC(obj_hash_empty);
break;
case 1:
RB_DEBUG_COUNTER_INC(obj_hash_1);
break;
case 2:
RB_DEBUG_COUNTER_INC(obj_hash_2);
break;
case 3:
RB_DEBUG_COUNTER_INC(obj_hash_3);
break;
case 4:
RB_DEBUG_COUNTER_INC(obj_hash_4);
break;
case 5:
case 6:
case 7:
case 8:
RB_DEBUG_COUNTER_INC(obj_hash_5_8);
break;
default:
GC_ASSERT(RHASH_SIZE(obj) > 8);
RB_DEBUG_COUNTER_INC(obj_hash_g8);
}
if (RHASH_AR_TABLE_P(obj)) {
if (RHASH_AR_TABLE(obj) == NULL) {
RB_DEBUG_COUNTER_INC(obj_hash_null);
}
else {
RB_DEBUG_COUNTER_INC(obj_hash_ar);
}
}
else {
RB_DEBUG_COUNTER_INC(obj_hash_st);
}
#endif
if (/* RHASH_AR_TABLE_P(obj) */ !FL_TEST_RAW(obj, RHASH_ST_TABLE_FLAG)) {
struct ar_table_struct *tab = RHASH(obj)->as.ar;
if (tab) {
if (RHASH_TRANSIENT_P(obj)) {
RB_DEBUG_COUNTER_INC(obj_hash_transient);
}
else {
ruby_xfree(tab);
}
}
}
else {
GC_ASSERT(RHASH_ST_TABLE_P(obj));
st_free_table(RHASH(obj)->as.st);
}
break;
case T_REGEXP:
if (RANY(obj)->as.regexp.ptr) {
onig_free(RANY(obj)->as.regexp.ptr);
RB_DEBUG_COUNTER_INC(obj_regexp_ptr);
}
break;
case T_DATA:
if (DATA_PTR(obj)) {
int free_immediately = FALSE;
void (*dfree)(void *);
void *data = DATA_PTR(obj);
if (RTYPEDDATA_P(obj)) {
free_immediately = (RANY(obj)->as.typeddata.type->flags & RUBY_TYPED_FREE_IMMEDIATELY) != 0;
dfree = RANY(obj)->as.typeddata.type->function.dfree;
if (0 && free_immediately == 0) {
/* to expose non-free-immediate T_DATA */
fprintf(stderr, "not immediate -> %s\n", RANY(obj)->as.typeddata.type->wrap_struct_name);
}
}
else {
dfree = RANY(obj)->as.data.dfree;
}
if (dfree) {
if (dfree == RUBY_DEFAULT_FREE) {
xfree(data);
RB_DEBUG_COUNTER_INC(obj_data_xfree);
}
else if (free_immediately) {
(*dfree)(data);
RB_DEBUG_COUNTER_INC(obj_data_imm_free);
}
else {
make_zombie(objspace, obj, dfree, data);
RB_DEBUG_COUNTER_INC(obj_data_zombie);
return 1;
}
}
else {
RB_DEBUG_COUNTER_INC(obj_data_empty);
}
}
break;
case T_MATCH:
if (RANY(obj)->as.match.rmatch) {
struct rmatch *rm = RANY(obj)->as.match.rmatch;
#if USE_DEBUG_COUNTER
if (rm->regs.num_regs >= 8) {
RB_DEBUG_COUNTER_INC(obj_match_ge8);
}
else if (rm->regs.num_regs >= 4) {
RB_DEBUG_COUNTER_INC(obj_match_ge4);
}
else if (rm->regs.num_regs >= 1) {
RB_DEBUG_COUNTER_INC(obj_match_under4);
}
#endif
onig_region_free(&rm->regs, 0);
if (rm->char_offset)
xfree(rm->char_offset);
xfree(rm);
RB_DEBUG_COUNTER_INC(obj_match_ptr);
}
break;
case T_FILE:
if (RANY(obj)->as.file.fptr) {
make_io_zombie(objspace, obj);
RB_DEBUG_COUNTER_INC(obj_file_ptr);
return 1;
}
break;
case T_RATIONAL:
RB_DEBUG_COUNTER_INC(obj_rational);
break;
case T_COMPLEX:
RB_DEBUG_COUNTER_INC(obj_complex);
break;
case T_MOVED:
break;
case T_ICLASS:
/* Basically , T_ICLASS shares table with the module */
if (FL_TEST(obj, RICLASS_IS_ORIGIN)) {
rb_id_table_free(RCLASS_M_TBL(obj));
}
if (RCLASS_CALLABLE_M_TBL(obj) != NULL) {
rb_id_table_free(RCLASS_CALLABLE_M_TBL(obj));
}
if (RCLASS_EXT(obj)->subclasses) {
rb_class_detach_subclasses(obj);
RCLASS_EXT(obj)->subclasses = NULL;
}
cc_table_free(objspace, obj, FALSE);
rb_class_remove_from_module_subclasses(obj);
rb_class_remove_from_super_subclasses(obj);
xfree(RANY(obj)->as.klass.ptr);
RANY(obj)->as.klass.ptr = NULL;
RB_DEBUG_COUNTER_INC(obj_iclass_ptr);
break;
case T_FLOAT:
RB_DEBUG_COUNTER_INC(obj_float);
break;
case T_BIGNUM:
if (!BIGNUM_EMBED_P(obj) && BIGNUM_DIGITS(obj)) {
xfree(BIGNUM_DIGITS(obj));
RB_DEBUG_COUNTER_INC(obj_bignum_ptr);
}
else {
RB_DEBUG_COUNTER_INC(obj_bignum_embed);
}
break;
case T_NODE:
UNEXPECTED_NODE(obj_free);
break;
case T_STRUCT:
if ((RBASIC(obj)->flags & RSTRUCT_EMBED_LEN_MASK) ||
RANY(obj)->as.rstruct.as.heap.ptr == NULL) {
RB_DEBUG_COUNTER_INC(obj_struct_embed);
}
else if (RSTRUCT_TRANSIENT_P(obj)) {
RB_DEBUG_COUNTER_INC(obj_struct_transient);
}
else {
xfree((void *)RANY(obj)->as.rstruct.as.heap.ptr);
RB_DEBUG_COUNTER_INC(obj_struct_ptr);
}
break;
case T_SYMBOL:
{
rb_gc_free_dsymbol(obj);
RB_DEBUG_COUNTER_INC(obj_symbol);
}
break;
case T_IMEMO:
switch (imemo_type(obj)) {
case imemo_ment:
rb_free_method_entry(&RANY(obj)->as.imemo.ment);
RB_DEBUG_COUNTER_INC(obj_imemo_ment);
break;
case imemo_iseq:
rb_iseq_free(&RANY(obj)->as.imemo.iseq);
RB_DEBUG_COUNTER_INC(obj_imemo_iseq);
break;
case imemo_env:
GC_ASSERT(VM_ENV_ESCAPED_P(RANY(obj)->as.imemo.env.ep));
xfree((VALUE *)RANY(obj)->as.imemo.env.env);
RB_DEBUG_COUNTER_INC(obj_imemo_env);
break;
case imemo_tmpbuf:
xfree(RANY(obj)->as.imemo.alloc.ptr);
RB_DEBUG_COUNTER_INC(obj_imemo_tmpbuf);
break;
case imemo_ast:
rb_ast_free(&RANY(obj)->as.imemo.ast);
RB_DEBUG_COUNTER_INC(obj_imemo_ast);
break;
case imemo_cref:
RB_DEBUG_COUNTER_INC(obj_imemo_cref);
break;
case imemo_svar:
RB_DEBUG_COUNTER_INC(obj_imemo_svar);
break;
case imemo_throw_data:
RB_DEBUG_COUNTER_INC(obj_imemo_throw_data);
break;
case imemo_ifunc:
RB_DEBUG_COUNTER_INC(obj_imemo_ifunc);
break;
case imemo_memo:
RB_DEBUG_COUNTER_INC(obj_imemo_memo);
break;
case imemo_parser_strterm:
RB_DEBUG_COUNTER_INC(obj_imemo_parser_strterm);
break;
case imemo_callinfo:
RB_DEBUG_COUNTER_INC(obj_imemo_callinfo);
break;
case imemo_callcache:
RB_DEBUG_COUNTER_INC(obj_imemo_callcache);
break;
default:
/* unreachable */
break;
}
return 0;
default:
rb_bug("gc_sweep(): unknown data type 0x%x(%p) 0x%"PRIxVALUE,
BUILTIN_TYPE(obj), (void*)obj, RBASIC(obj)->flags);
}
if (FL_TEST(obj, FL_FINALIZE)) {
make_zombie(objspace, obj, 0, 0);
return 1;
}
else {
return 0;
}
}
#define OBJ_ID_INCREMENT (sizeof(RVALUE) / 2)
#define OBJ_ID_INITIAL (OBJ_ID_INCREMENT * 2)
static int
object_id_cmp(st_data_t x, st_data_t y)
{
if (RB_TYPE_P(x, T_BIGNUM)) {
return !rb_big_eql(x, y);
} else {
return x != y;
}
}
static st_index_t
object_id_hash(st_data_t n)
{
if (RB_TYPE_P(n, T_BIGNUM)) {
return FIX2LONG(rb_big_hash(n));
} else {
return st_numhash(n);
}
}
static const struct st_hash_type object_id_hash_type = {
object_id_cmp,
object_id_hash,
};
void
Init_heap(void)
{
rb_objspace_t *objspace = &rb_objspace;
objspace->next_object_id = INT2FIX(OBJ_ID_INITIAL);
objspace->id_to_obj_tbl = st_init_table(&object_id_hash_type);
objspace->obj_to_id_tbl = st_init_numtable();
#if RGENGC_ESTIMATE_OLDMALLOC
objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
#endif
heap_add_pages(objspace, heap_eden, gc_params.heap_init_slots / HEAP_PAGE_OBJ_LIMIT);
init_mark_stack(&objspace->mark_stack);
objspace->profile.invoke_time = getrusage_time();
finalizer_table = st_init_numtable();
}
void
Init_gc_stress(void)
{
rb_objspace_t *objspace = &rb_objspace;
gc_stress_set(objspace, ruby_initial_gc_stress);
}
typedef int each_obj_callback(void *, void *, size_t, void *);
static void objspace_each_objects(rb_objspace_t *objspace, each_obj_callback *callback, void *data);
static void objspace_reachable_objects_from_root(rb_objspace_t *, void (func)(const char *, VALUE, void *), void *);
struct each_obj_args {
rb_objspace_t *objspace;
each_obj_callback *callback;
void *data;
};
static void
objspace_each_objects_without_setup(rb_objspace_t *objspace, each_obj_callback *callback, void *data)
{
size_t i;
struct heap_page *page;
RVALUE *pstart = NULL, *pend;
i = 0;
while (i < heap_allocated_pages) {
while (0 < i && pstart < heap_pages_sorted[i-1]->start) i--;
while (i < heap_allocated_pages && heap_pages_sorted[i]->start <= pstart) i++;
if (heap_allocated_pages <= i) break;
page = heap_pages_sorted[i];
pstart = page->start;
pend = pstart + page->total_slots;
if ((*callback)(pstart, pend, sizeof(RVALUE), data)) {
break;
}
}
}
static VALUE
objspace_each_objects_protected(VALUE arg)
{
struct each_obj_args *args = (struct each_obj_args *)arg;
objspace_each_objects_without_setup(args->objspace, args->callback, args->data);
return Qnil;
}
static VALUE
incremental_enable(VALUE _)
{
rb_objspace_t *objspace = &rb_objspace;
objspace->flags.dont_incremental = FALSE;
return Qnil;
}
/*
* rb_objspace_each_objects() is special C API to walk through
* Ruby object space. This C API is too difficult to use it.
* To be frank, you should not use it. Or you need to read the
* source code of this function and understand what this function does.
*
* 'callback' will be called several times (the number of heap page,
* at current implementation) with:
* vstart: a pointer to the first living object of the heap_page.
* vend: a pointer to next to the valid heap_page area.
* stride: a distance to next VALUE.
*
* If callback() returns non-zero, the iteration will be stopped.
*
* This is a sample callback code to iterate liveness objects:
*
* int
* sample_callback(void *vstart, void *vend, int stride, void *data) {
* VALUE v = (VALUE)vstart;
* for (; v != (VALUE)vend; v += stride) {
* if (RBASIC(v)->flags) { // liveness check
* // do something with live object 'v'
* }
* return 0; // continue to iteration
* }
*
* Note: 'vstart' is not a top of heap_page. This point the first
* living object to grasp at least one object to avoid GC issue.
* This means that you can not walk through all Ruby object page
* including freed object page.
*
* Note: On this implementation, 'stride' is same as sizeof(RVALUE).
* However, there are possibilities to pass variable values with
* 'stride' with some reasons. You must use stride instead of
* use some constant value in the iteration.
*/
void
rb_objspace_each_objects(each_obj_callback *callback, void *data)
{
objspace_each_objects(&rb_objspace, callback, data);
}
static void
objspace_each_objects(rb_objspace_t *objspace, each_obj_callback *callback, void *data)
{
int prev_dont_incremental = objspace->flags.dont_incremental;
gc_rest(objspace);
objspace->flags.d