Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
branch: trunk
9122 lines (7929 sloc) 242.157 kb
/**********************************************************************
gc.c -
$Author$
created at: Tue Oct 5 09:44:46 JST 1993
Copyright (C) 1993-2007 Yukihiro Matsumoto
Copyright (C) 2000 Network Applied Communication Laboratory, Inc.
Copyright (C) 2000 Information-technology Promotion Agency, Japan
**********************************************************************/
#include "internal.h"
#include "ruby/st.h"
#include "ruby/re.h"
#include "ruby/io.h"
#include "ruby/thread.h"
#include "ruby/util.h"
#include "ruby/debug.h"
#include "eval_intern.h"
#include "vm_core.h"
#include "gc.h"
#include "constant.h"
#include "ruby_atomic.h"
#include "probes.h"
#include <stdio.h>
#include <stdarg.h>
#include <setjmp.h>
#include <sys/types.h>
#include <assert.h>
#undef rb_data_object_alloc
#ifndef __has_feature
# define __has_feature(x) 0
#endif
#ifndef HAVE_MALLOC_USABLE_SIZE
# ifdef _WIN32
# define HAVE_MALLOC_USABLE_SIZE
# define malloc_usable_size(a) _msize(a)
# elif defined HAVE_MALLOC_SIZE
# define HAVE_MALLOC_USABLE_SIZE
# define malloc_usable_size(a) malloc_size(a)
# endif
#endif
#ifdef HAVE_MALLOC_USABLE_SIZE
# ifdef HAVE_MALLOC_H
# include <malloc.h>
# elif defined(HAVE_MALLOC_NP_H)
# include <malloc_np.h>
# elif defined(HAVE_MALLOC_MALLOC_H)
# include <malloc/malloc.h>
# endif
#endif
#if /* is ASAN enabled? */ \
__has_feature(address_sanitizer) /* Clang */ || \
defined(__SANITIZE_ADDRESS__) /* GCC 4.8.x */
#define ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS \
__attribute__((no_address_safety_analysis)) \
__attribute__((noinline))
#else
#define ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS
#endif
#ifdef HAVE_SYS_TIME_H
#include <sys/time.h>
#endif
#ifdef HAVE_SYS_RESOURCE_H
#include <sys/resource.h>
#endif
#if defined(__native_client__) && defined(NACL_NEWLIB)
# include "nacl/resource.h"
# undef HAVE_POSIX_MEMALIGN
# undef HAVE_MEMALIGN
#endif
#if defined _WIN32 || defined __CYGWIN__
#include <windows.h>
#elif defined(HAVE_POSIX_MEMALIGN)
#elif defined(HAVE_MEMALIGN)
#include <malloc.h>
#endif
#define rb_setjmp(env) RUBY_SETJMP(env)
#define rb_jmp_buf rb_jmpbuf_t
#if defined(HAVE_RB_GC_GUARDED_PTR_VAL) && HAVE_RB_GC_GUARDED_PTR_VAL
/* trick the compiler into thinking a external signal handler uses this */
volatile VALUE rb_gc_guarded_val;
volatile VALUE *
rb_gc_guarded_ptr_val(volatile VALUE *ptr, VALUE val)
{
rb_gc_guarded_val = val;
return ptr;
}
#endif
#ifndef GC_HEAP_INIT_SLOTS
#define GC_HEAP_INIT_SLOTS 10000
#endif
#ifndef GC_HEAP_FREE_SLOTS
#define GC_HEAP_FREE_SLOTS 4096
#endif
#ifndef GC_HEAP_GROWTH_FACTOR
#define GC_HEAP_GROWTH_FACTOR 1.8
#endif
#ifndef GC_HEAP_GROWTH_MAX_SLOTS
#define GC_HEAP_GROWTH_MAX_SLOTS 0 /* 0 is disable */
#endif
#ifndef GC_HEAP_OLDOBJECT_LIMIT_FACTOR
#define GC_HEAP_OLDOBJECT_LIMIT_FACTOR 2.0
#endif
#ifndef GC_HEAP_FREE_SLOTS_MIN_RATIO
#define GC_HEAP_FREE_SLOTS_MIN_RATIO 0.3
#endif
#ifndef GC_HEAP_FREE_SLOTS_MAX_RATIO
#define GC_HEAP_FREE_SLOTS_MAX_RATIO 0.8
#endif
#ifndef GC_MALLOC_LIMIT_MIN
#define GC_MALLOC_LIMIT_MIN (16 * 1024 * 1024 /* 16MB */)
#endif
#ifndef GC_MALLOC_LIMIT_MAX
#define GC_MALLOC_LIMIT_MAX (32 * 1024 * 1024 /* 32MB */)
#endif
#ifndef GC_MALLOC_LIMIT_GROWTH_FACTOR
#define GC_MALLOC_LIMIT_GROWTH_FACTOR 1.4
#endif
#ifndef GC_OLDMALLOC_LIMIT_MIN
#define GC_OLDMALLOC_LIMIT_MIN (16 * 1024 * 1024 /* 16MB */)
#endif
#ifndef GC_OLDMALLOC_LIMIT_GROWTH_FACTOR
#define GC_OLDMALLOC_LIMIT_GROWTH_FACTOR 1.2
#endif
#ifndef GC_OLDMALLOC_LIMIT_MAX
#define GC_OLDMALLOC_LIMIT_MAX (128 * 1024 * 1024 /* 128MB */)
#endif
#ifndef PRINT_MEASURE_LINE
#define PRINT_MEASURE_LINE 0
#endif
#ifndef PRINT_ENTER_EXIT_TICK
#define PRINT_ENTER_EXIT_TICK 0
#endif
#ifndef PRINT_ROOT_TICKS
#define PRINT_ROOT_TICKS 0
#endif
#define USE_TICK_T (PRINT_ENTER_EXIT_TICK || PRINT_MEASURE_LINE)
#define TICK_TYPE 1
typedef struct {
size_t heap_init_slots;
size_t heap_free_slots;
double growth_factor;
size_t growth_max_slots;
double oldobject_limit_factor;
size_t malloc_limit_min;
size_t malloc_limit_max;
double malloc_limit_growth_factor;
size_t oldmalloc_limit_min;
size_t oldmalloc_limit_max;
double oldmalloc_limit_growth_factor;
VALUE gc_stress;
} ruby_gc_params_t;
static ruby_gc_params_t gc_params = {
GC_HEAP_INIT_SLOTS,
GC_HEAP_FREE_SLOTS,
GC_HEAP_GROWTH_FACTOR,
GC_HEAP_GROWTH_MAX_SLOTS,
GC_HEAP_OLDOBJECT_LIMIT_FACTOR,
GC_MALLOC_LIMIT_MIN,
GC_MALLOC_LIMIT_MAX,
GC_MALLOC_LIMIT_GROWTH_FACTOR,
GC_OLDMALLOC_LIMIT_MIN,
GC_OLDMALLOC_LIMIT_MAX,
GC_OLDMALLOC_LIMIT_GROWTH_FACTOR,
#if defined(ENABLE_VM_OBJSPACE) && ENABLE_VM_OBJSPACE
FALSE,
#endif
};
/* GC_DEBUG:
* enable to embed GC debugging information.
*/
#ifndef GC_DEBUG
#define GC_DEBUG 0
#endif
#if USE_RGENGC
/* RGENGC_DEBUG:
* 1: basic information
* 2: remember set operation
* 3: mark
* 4:
* 5: sweep
*/
#ifndef RGENGC_DEBUG
#define RGENGC_DEBUG 0
#endif
/* RGENGC_CHECK_MODE
* 0: disable all assertions
* 1: enable assertions (to debug RGenGC)
* 2: enable internal consistency check at each GC (for debugging)
* 3: enable internal consistency check at each GC steps (for debugging)
* 4: enable livness check
* 5: show all references
*/
#ifndef RGENGC_CHECK_MODE
#define RGENGC_CHECK_MODE 0
#endif
/* RGENGC_OLD_NEWOBJ_CHECK
* 0: disable all assertions
* >0: make a OLD object when new object creation.
*
* Make one OLD object per RGENGC_OLD_NEWOBJ_CHECK WB protected objects creation.
*/
#ifndef RGENGC_OLD_NEWOBJ_CHECK
#define RGENGC_OLD_NEWOBJ_CHECK 0
#endif
/* RGENGC_PROFILE
* 0: disable RGenGC profiling
* 1: enable profiling for basic information
* 2: enable profiling for each types
*/
#ifndef RGENGC_PROFILE
#define RGENGC_PROFILE 0
#endif
/* RGENGC_ESTIMATE_OLDMALLOC
* Enable/disable to estimate increase size of malloc'ed size by old objects.
* If estimation exceeds threshold, then will invoke full GC.
* 0: disable estimation.
* 1: enable estimation.
*/
#ifndef RGENGC_ESTIMATE_OLDMALLOC
#define RGENGC_ESTIMATE_OLDMALLOC 1
#endif
/* RGENGC_FORCE_MAJOR_GC
* Force major/full GC if this macro is not 0.
*/
#ifndef RGENGC_FORCE_MAJOR_GC
#define RGENGC_FORCE_MAJOR_GC 0
#endif
#else /* USE_RGENGC */
#ifdef RGENGC_DEBUG
#undef RGENGC_DEBUG
#endif
#define RGENGC_DEBUG 0
#ifdef RGENGC_CHECK_MODE
#undef RGENGC_CHECK_MODE
#endif
#define RGENGC_CHECK_MODE 0
#define RGENGC_PROFILE 0
#define RGENGC_ESTIMATE_OLDMALLOC 0
#define RGENGC_FORCE_MAJOR_GC 0
#endif /* USE_RGENGC */
#ifndef GC_PROFILE_MORE_DETAIL
#define GC_PROFILE_MORE_DETAIL 0
#endif
#ifndef GC_PROFILE_DETAIL_MEMORY
#define GC_PROFILE_DETAIL_MEMORY 0
#endif
#ifndef GC_ENABLE_INCREMENTAL_MARK
#define GC_ENABLE_INCREMENTAL_MARK USE_RINCGC
#endif
#ifndef GC_ENABLE_LAZY_SWEEP
#define GC_ENABLE_LAZY_SWEEP 1
#endif
#ifndef CALC_EXACT_MALLOC_SIZE
#define CALC_EXACT_MALLOC_SIZE 0
#endif
#if defined(HAVE_MALLOC_USABLE_SIZE) || CALC_EXACT_MALLOC_SIZE > 0
#ifndef MALLOC_ALLOCATED_SIZE
#define MALLOC_ALLOCATED_SIZE 0
#endif
#else
#define MALLOC_ALLOCATED_SIZE 0
#endif
#ifndef MALLOC_ALLOCATED_SIZE_CHECK
#define MALLOC_ALLOCATED_SIZE_CHECK 0
#endif
typedef enum {
GPR_FLAG_NONE = 0x000,
/* major reason */
GPR_FLAG_MAJOR_BY_NOFREE = 0x001,
GPR_FLAG_MAJOR_BY_OLDGEN = 0x002,
GPR_FLAG_MAJOR_BY_SHADY = 0x004,
GPR_FLAG_MAJOR_BY_FORCE = 0x008,
#if RGENGC_ESTIMATE_OLDMALLOC
GPR_FLAG_MAJOR_BY_OLDMALLOC = 0x020,
#endif
GPR_FLAG_MAJOR_MASK = 0x0ff,
/* gc reason */
GPR_FLAG_NEWOBJ = 0x100,
GPR_FLAG_MALLOC = 0x200,
GPR_FLAG_METHOD = 0x400,
GPR_FLAG_CAPI = 0x800,
GPR_FLAG_STRESS = 0x1000,
/* others */
GPR_FLAG_IMMEDIATE_SWEEP = 0x2000,
GPR_FLAG_HAVE_FINALIZE = 0x4000
} gc_profile_record_flag;
typedef struct gc_profile_record {
int flags;
double gc_time;
double gc_invoke_time;
size_t heap_total_objects;
size_t heap_use_size;
size_t heap_total_size;
#if GC_PROFILE_MORE_DETAIL
double gc_mark_time;
double gc_sweep_time;
size_t heap_use_pages;
size_t heap_live_objects;
size_t heap_free_objects;
size_t allocate_increase;
size_t allocate_limit;
double prepare_time;
size_t removing_objects;
size_t empty_objects;
#if GC_PROFILE_DETAIL_MEMORY
long maxrss;
long minflt;
long majflt;
#endif
#endif
#if MALLOC_ALLOCATED_SIZE
size_t allocated_size;
#endif
#if RGENGC_PROFILE > 0
size_t old_objects;
size_t remembered_normal_objects;
size_t remembered_shady_objects;
#endif
} gc_profile_record;
#if defined(_MSC_VER) || defined(__BORLANDC__) || defined(__CYGWIN__)
#pragma pack(push, 1) /* magic for reducing sizeof(RVALUE): 24 -> 20 */
#endif
typedef struct RVALUE {
union {
struct {
VALUE flags; /* always 0 for freed obj */
struct RVALUE *next;
} free;
struct RBasic basic;
struct RObject object;
struct RClass klass;
struct RFloat flonum;
struct RString string;
struct RArray array;
struct RRegexp regexp;
struct RHash hash;
struct RData data;
struct RTypedData typeddata;
struct RStruct rstruct;
struct RBignum bignum;
struct RFile file;
struct RNode node;
struct RMatch match;
struct RRational rational;
struct RComplex complex;
union {
rb_cref_t cref;
struct vm_svar svar;
struct vm_throw_data throw_data;
struct vm_ifunc ifunc;
struct MEMO memo;
} imemo;
struct {
struct RBasic basic;
VALUE v1;
VALUE v2;
VALUE v3;
} values;
} as;
#if GC_DEBUG
const char *file;
int line;
#endif
} RVALUE;
#if defined(_MSC_VER) || defined(__BORLANDC__) || defined(__CYGWIN__)
#pragma pack(pop)
#endif
typedef uintptr_t bits_t;
enum {
BITS_SIZE = sizeof(bits_t),
BITS_BITLENGTH = ( BITS_SIZE * CHAR_BIT )
};
struct heap_page_header {
struct heap_page *page;
};
struct heap_page_body {
struct heap_page_header header;
/* char gap[]; */
/* RVALUE values[]; */
};
struct gc_list {
VALUE *varptr;
struct gc_list *next;
};
#define STACK_CHUNK_SIZE 500
typedef struct stack_chunk {
VALUE data[STACK_CHUNK_SIZE];
struct stack_chunk *next;
} stack_chunk_t;
typedef struct mark_stack {
stack_chunk_t *chunk;
stack_chunk_t *cache;
int index;
int limit;
size_t cache_size;
size_t unused_cache_size;
} mark_stack_t;
typedef struct rb_heap_struct {
RVALUE *freelist;
struct heap_page *free_pages;
struct heap_page *using_page;
struct heap_page *pages;
struct heap_page *sweep_pages;
#if GC_ENABLE_INCREMENTAL_MARK
struct heap_page *pooled_pages;
#endif
size_t page_length; /* total page count in a heap */
size_t total_slots; /* total slot count (page_length * HEAP_OBJ_LIMIT) */
} rb_heap_t;
enum gc_stat {
gc_stat_none,
gc_stat_marking,
gc_stat_sweeping
};
typedef struct rb_objspace {
struct {
size_t limit;
size_t increase;
#if MALLOC_ALLOCATED_SIZE
size_t allocated_size;
size_t allocations;
#endif
} malloc_params;
struct {
enum gc_stat stat : 2;
unsigned int immediate_sweep : 1;
unsigned int dont_gc : 1;
unsigned int dont_incremental : 1;
unsigned int during_gc : 1;
unsigned int gc_stressful: 1;
#if USE_RGENGC
unsigned int during_minor_gc : 1;
#endif
#if GC_ENABLE_INCREMENTAL_MARK
unsigned int during_incremental_marking : 1;
#endif
} flags;
rb_event_flag_t hook_events;
size_t total_allocated_objects;
rb_heap_t eden_heap;
rb_heap_t tomb_heap; /* heap for zombies and ghosts */
struct {
rb_atomic_t finalizing;
} atomic_flags;
struct mark_func_data_struct {
void *data;
void (*mark_func)(VALUE v, void *data);
} *mark_func_data;
mark_stack_t mark_stack;
size_t marked_slots;
struct {
struct heap_page **sorted;
size_t allocated_pages;
size_t allocatable_pages;
size_t sorted_length;
RVALUE *range[2];
size_t swept_slots;
size_t min_free_slots;
size_t max_free_slots;
/* final */
size_t final_slots;
VALUE deferred_final;
} heap_pages;
st_table *finalizer_table;
struct {
int run;
int latest_gc_info;
gc_profile_record *records;
gc_profile_record *current_record;
size_t next_index;
size_t size;
#if GC_PROFILE_MORE_DETAIL
double prepare_time;
#endif
double invoke_time;
#if USE_RGENGC
size_t minor_gc_count;
size_t major_gc_count;
#if RGENGC_PROFILE > 0
size_t total_generated_normal_object_count;
size_t total_generated_shady_object_count;
size_t total_shade_operation_count;
size_t total_promoted_count;
size_t total_remembered_normal_object_count;
size_t total_remembered_shady_object_count;
#if RGENGC_PROFILE >= 2
size_t generated_normal_object_count_types[RUBY_T_MASK];
size_t generated_shady_object_count_types[RUBY_T_MASK];
size_t shade_operation_count_types[RUBY_T_MASK];
size_t promoted_types[RUBY_T_MASK];
size_t remembered_normal_object_count_types[RUBY_T_MASK];
size_t remembered_shady_object_count_types[RUBY_T_MASK];
#endif
#endif /* RGENGC_PROFILE */
#endif /* USE_RGENGC */
/* temporary profiling space */
double gc_sweep_start_time;
size_t total_allocated_objects_at_gc_start;
size_t heap_used_at_gc_start;
/* basic statistics */
size_t count;
size_t total_freed_objects;
size_t total_allocated_pages;
size_t total_freed_pages;
} profile;
struct gc_list *global_list;
VALUE gc_stress_mode;
#if USE_RGENGC
struct {
VALUE parent_object;
int need_major_gc;
size_t last_major_gc;
size_t uncollectible_wb_unprotected_objects;
size_t uncollectible_wb_unprotected_objects_limit;
size_t old_objects;
size_t old_objects_limit;
#if RGENGC_ESTIMATE_OLDMALLOC
size_t oldmalloc_increase;
size_t oldmalloc_increase_limit;
#endif
#if RGENGC_CHECK_MODE >= 2
struct st_table *allrefs_table;
size_t error_count;
#endif
} rgengc;
#if GC_ENABLE_INCREMENTAL_MARK
struct {
size_t pooled_slots;
size_t step_slots;
} rincgc;
#endif
#endif /* USE_RGENGC */
} rb_objspace_t;
#ifndef HEAP_ALIGN_LOG
/* default tiny heap size: 16KB */
#define HEAP_ALIGN_LOG 14
#endif
#define CEILDIV(i, mod) (((i) + (mod) - 1)/(mod))
enum {
HEAP_ALIGN = (1UL << HEAP_ALIGN_LOG),
HEAP_ALIGN_MASK = (~(~0UL << HEAP_ALIGN_LOG)),
REQUIRED_SIZE_BY_MALLOC = (sizeof(size_t) * 5),
HEAP_SIZE = (HEAP_ALIGN - REQUIRED_SIZE_BY_MALLOC),
HEAP_OBJ_LIMIT = (unsigned int)((HEAP_SIZE - sizeof(struct heap_page_header))/sizeof(struct RVALUE)),
HEAP_BITMAP_LIMIT = CEILDIV(CEILDIV(HEAP_SIZE, sizeof(struct RVALUE)), BITS_BITLENGTH),
HEAP_BITMAP_SIZE = ( BITS_SIZE * HEAP_BITMAP_LIMIT),
HEAP_BITMAP_PLANES = USE_RGENGC ? 3 : 1 /* RGENGC: mark bits, rememberset bits and oldgen bits */
};
struct heap_page {
struct heap_page_body *body;
struct heap_page *prev;
rb_heap_t *heap;
int total_slots;
int free_slots;
int final_slots;
struct {
unsigned int before_sweep : 1;
unsigned int has_remembered_objects : 1;
unsigned int has_uncollectible_shady_objects : 1;
} flags;
struct heap_page *free_next;
RVALUE *start;
RVALUE *freelist;
struct heap_page *next;
#if USE_RGENGC
bits_t wb_unprotected_bits[HEAP_BITMAP_LIMIT];
#endif
/* the following three bitmaps are cleared at the beggining of full GC */
bits_t mark_bits[HEAP_BITMAP_LIMIT];
#if USE_RGENGC
bits_t uncollectible_bits[HEAP_BITMAP_LIMIT];
bits_t marking_bits[HEAP_BITMAP_LIMIT];
#endif
};
#define GET_PAGE_BODY(x) ((struct heap_page_body *)((bits_t)(x) & ~(HEAP_ALIGN_MASK)))
#define GET_PAGE_HEADER(x) (&GET_PAGE_BODY(x)->header)
#define GET_HEAP_PAGE(x) (GET_PAGE_HEADER(x)->page)
#define NUM_IN_PAGE(p) (((bits_t)(p) & HEAP_ALIGN_MASK)/sizeof(RVALUE))
#define BITMAP_INDEX(p) (NUM_IN_PAGE(p) / BITS_BITLENGTH )
#define BITMAP_OFFSET(p) (NUM_IN_PAGE(p) & (BITS_BITLENGTH-1))
#define BITMAP_BIT(p) ((bits_t)1 << BITMAP_OFFSET(p))
/* Bitmap Operations */
#define MARKED_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] & BITMAP_BIT(p))
#define MARK_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] = (bits)[BITMAP_INDEX(p)] | BITMAP_BIT(p))
#define CLEAR_IN_BITMAP(bits, p) ((bits)[BITMAP_INDEX(p)] = (bits)[BITMAP_INDEX(p)] & ~BITMAP_BIT(p))
/* getting bitmap */
#define GET_HEAP_MARK_BITS(x) (&GET_HEAP_PAGE(x)->mark_bits[0])
#if USE_RGENGC
#define GET_HEAP_UNCOLLECTIBLE_BITS(x) (&GET_HEAP_PAGE(x)->uncollectible_bits[0])
#define GET_HEAP_WB_UNPROTECTED_BITS(x) (&GET_HEAP_PAGE(x)->wb_unprotected_bits[0])
#define GET_HEAP_MARKING_BITS(x) (&GET_HEAP_PAGE(x)->marking_bits[0])
#endif
/* Aliases */
#if defined(ENABLE_VM_OBJSPACE) && ENABLE_VM_OBJSPACE
#define rb_objspace (*GET_VM()->objspace)
#else
static rb_objspace_t rb_objspace = {{GC_MALLOC_LIMIT_MIN}};
#endif
#define ruby_initial_gc_stress gc_params.gc_stress
VALUE *ruby_initial_gc_stress_ptr = &ruby_initial_gc_stress;
#define malloc_limit objspace->malloc_params.limit
#define malloc_increase objspace->malloc_params.increase
#define malloc_allocated_size objspace->malloc_params.allocated_size
#define heap_pages_sorted objspace->heap_pages.sorted
#define heap_allocated_pages objspace->heap_pages.allocated_pages
#define heap_pages_sorted_length objspace->heap_pages.sorted_length
#define heap_pages_lomem objspace->heap_pages.range[0]
#define heap_pages_himem objspace->heap_pages.range[1]
#define heap_pages_swept_slots objspace->heap_pages.swept_slots
#define heap_allocatable_pages objspace->heap_pages.allocatable_pages
#define heap_pages_min_free_slots objspace->heap_pages.min_free_slots
#define heap_pages_max_free_slots objspace->heap_pages.max_free_slots
#define heap_pages_final_slots objspace->heap_pages.final_slots
#define heap_pages_deferred_final objspace->heap_pages.deferred_final
#define heap_eden (&objspace->eden_heap)
#define heap_tomb (&objspace->tomb_heap)
#define dont_gc objspace->flags.dont_gc
#define during_gc objspace->flags.during_gc
#define finalizing objspace->atomic_flags.finalizing
#define finalizer_table objspace->finalizer_table
#define global_list objspace->global_list
#define ruby_gc_stressful objspace->flags.gc_stressful
#define ruby_gc_stress_mode objspace->gc_stress_mode
#define is_marking(objspace) ((objspace)->flags.stat == gc_stat_marking)
#define is_sweeping(objspace) ((objspace)->flags.stat == gc_stat_sweeping)
#if USE_RGENGC
#define is_full_marking(objspace) ((objspace)->flags.during_minor_gc == FALSE)
#else
#define is_full_marking(objspace) TRUE
#endif
#if GC_ENABLE_INCREMENTAL_MARK
#define is_incremental_marking(objspace) ((objspace)->flags.during_incremental_marking != FALSE)
#else
#define is_incremental_marking(objspace) FALSE
#endif
#if GC_ENABLE_INCREMENTAL_MARK
#define will_be_incremental_marking(objspace) ((objspace)->rgengc.need_major_gc != GPR_FLAG_NONE)
#else
#define will_be_incremental_marking(objspace) FALSE
#endif
#define has_sweeping_pages(heap) ((heap)->sweep_pages != 0)
#define is_lazy_sweeping(heap) (GC_ENABLE_LAZY_SWEEP && has_sweeping_pages(heap))
#if SIZEOF_LONG == SIZEOF_VOIDP
# define nonspecial_obj_id(obj) (VALUE)((SIGNED_VALUE)(obj)|FIXNUM_FLAG)
# define obj_id_to_ref(objid) ((objid) ^ FIXNUM_FLAG) /* unset FIXNUM_FLAG */
#elif SIZEOF_LONG_LONG == SIZEOF_VOIDP
# define nonspecial_obj_id(obj) LL2NUM((SIGNED_VALUE)(obj) / 2)
# define obj_id_to_ref(objid) (FIXNUM_P(objid) ? \
((objid) ^ FIXNUM_FLAG) : (NUM2PTR(objid) << 1))
#else
# error not supported
#endif
#define RANY(o) ((RVALUE*)(o))
struct RZombie {
struct RBasic basic;
VALUE next;
void (*dfree)(void *);
void *data;
};
#define RZOMBIE(o) ((struct RZombie *)(o))
#define nomem_error GET_VM()->special_exceptions[ruby_error_nomemory]
int ruby_gc_debug_indent = 0;
VALUE rb_mGC;
int ruby_disable_gc = 0;
void rb_gcdebug_print_obj_condition(VALUE obj);
static void rb_objspace_call_finalizer(rb_objspace_t *objspace);
static VALUE define_final0(VALUE obj, VALUE block);
static void negative_size_allocation_error(const char *);
static void *aligned_malloc(size_t, size_t);
static void aligned_free(void *);
static void init_mark_stack(mark_stack_t *stack);
static int ready_to_gc(rb_objspace_t *objspace);
static int garbage_collect(rb_objspace_t *, int full_mark, int immediate_mark, int immediate_sweep, int reason);
static int gc_start(rb_objspace_t *objspace, const int full_mark, const int immediate_mark, const unsigned int immediate_sweep, int reason);
static void gc_rest(rb_objspace_t *objspace);
static inline void gc_enter(rb_objspace_t *objspace, const char *event);
static inline void gc_exit(rb_objspace_t *objspace, const char *event);
static void gc_marks(rb_objspace_t *objspace, int full_mark);
static void gc_marks_start(rb_objspace_t *objspace, int full);
static int gc_marks_finish(rb_objspace_t *objspace);
static void gc_marks_rest(rb_objspace_t *objspace);
#if GC_ENABLE_INCREMENTAL_MARK
static void gc_marks_step(rb_objspace_t *objspace, int slots);
static void gc_marks_continue(rb_objspace_t *objspace, rb_heap_t *heap);
#endif
static void gc_sweep(rb_objspace_t *objspace);
static void gc_sweep_start(rb_objspace_t *objspace);
static void gc_sweep_finish(rb_objspace_t *objspace);
static int gc_sweep_step(rb_objspace_t *objspace, rb_heap_t *heap);
static void gc_sweep_rest(rb_objspace_t *objspace);
#if GC_ENABLE_LAZY_SWEEP
static void gc_sweep_continue(rb_objspace_t *objspace, rb_heap_t *heap);
#endif
static void gc_mark(rb_objspace_t *objspace, VALUE ptr);
static void gc_mark_ptr(rb_objspace_t *objspace, VALUE ptr);
static void gc_mark_maybe(rb_objspace_t *objspace, VALUE ptr);
static void gc_mark_children(rb_objspace_t *objspace, VALUE ptr);
static int gc_mark_stacked_objects_incremental(rb_objspace_t *, size_t count);
static int gc_mark_stacked_objects_all(rb_objspace_t *);
static void gc_grey(rb_objspace_t *objspace, VALUE ptr);
static inline int gc_mark_set(rb_objspace_t *objspace, VALUE obj);
static inline int is_pointer_to_heap(rb_objspace_t *objspace, void *ptr);
static void push_mark_stack(mark_stack_t *, VALUE);
static int pop_mark_stack(mark_stack_t *, VALUE *);
static size_t mark_stack_size(mark_stack_t *stack);
static void shrink_stack_chunk_cache(mark_stack_t *stack);
static size_t obj_memsize_of(VALUE obj, int use_all_types);
static VALUE gc_verify_internal_consistency(VALUE self);
static int gc_verify_heap_page(rb_objspace_t *objspace, struct heap_page *page, VALUE obj);
static int gc_verify_heap_pages(rb_objspace_t *objspace);
static void gc_stress_set(rb_objspace_t *objspace, VALUE flag);
static double getrusage_time(void);
static inline void gc_prof_setup_new_record(rb_objspace_t *objspace, int reason);
static inline void gc_prof_timer_start(rb_objspace_t *);
static inline void gc_prof_timer_stop(rb_objspace_t *);
static inline void gc_prof_mark_timer_start(rb_objspace_t *);
static inline void gc_prof_mark_timer_stop(rb_objspace_t *);
static inline void gc_prof_sweep_timer_start(rb_objspace_t *);
static inline void gc_prof_sweep_timer_stop(rb_objspace_t *);
static inline void gc_prof_set_malloc_info(rb_objspace_t *);
static inline void gc_prof_set_heap_info(rb_objspace_t *);
#define gc_prof_record(objspace) (objspace)->profile.current_record
#define gc_prof_enabled(objspace) ((objspace)->profile.run && (objspace)->profile.current_record)
#ifdef HAVE_VA_ARGS_MACRO
# define gc_report(level, objspace, fmt, ...) \
if ((level) > RGENGC_DEBUG) {} else gc_report_body(level, objspace, fmt, ##__VA_ARGS__)
#else
# define gc_report if (!(RGENGC_DEBUG)) {} else gc_report_body
#endif
PRINTF_ARGS(static void gc_report_body(int level, rb_objspace_t *objspace, const char *fmt, ...), 3, 4);
static const char *obj_info(VALUE obj);
#define PUSH_MARK_FUNC_DATA(v) do { \
struct mark_func_data_struct *prev_mark_func_data = objspace->mark_func_data; \
objspace->mark_func_data = (v);
#define POP_MARK_FUNC_DATA() objspace->mark_func_data = prev_mark_func_data;} while (0)
/*
* 1 - TSC (H/W Time Stamp Counter)
* 2 - getrusage
*/
#ifndef TICK_TYPE
#define TICK_TYPE 1
#endif
#if USE_TICK_T
#if TICK_TYPE == 1
/* the following code is only for internal tuning. */
/* Source code to use RDTSC is quoted and modified from
* http://www.mcs.anl.gov/~kazutomo/rdtsc.html
* written by Kazutomo Yoshii <kazutomo@mcs.anl.gov>
*/
#if defined(__GNUC__) && defined(__i386__)
typedef unsigned long long tick_t;
#define PRItick "llu"
static inline tick_t
tick(void)
{
unsigned long long int x;
__asm__ __volatile__ ("rdtsc" : "=A" (x));
return x;
}
#elif defined(__GNUC__) && defined(__x86_64__)
typedef unsigned long long tick_t;
#define PRItick "llu"
static __inline__ tick_t
tick(void)
{
unsigned long hi, lo;
__asm__ __volatile__ ("rdtsc" : "=a"(lo), "=d"(hi));
return ((unsigned long long)lo)|( ((unsigned long long)hi)<<32);
}
#elif defined(_WIN32) && defined(_MSC_VER)
#include <intrin.h>
typedef unsigned __int64 tick_t;
#define PRItick "llu"
static inline tick_t
tick(void)
{
return __rdtsc();
}
#else /* use clock */
typedef clock_t tick_t;
#define PRItick "llu"
static inline tick_t
tick(void)
{
return clock();
}
#endif /* TSC */
#elif TICK_TYPE == 2
typedef double tick_t;
#define PRItick "4.9f"
static inline tick_t
tick(void)
{
return getrusage_time();
}
#else /* TICK_TYPE */
#error "choose tick type"
#endif /* TICK_TYPE */
#define MEASURE_LINE(expr) do { \
volatile tick_t start_time = tick(); \
volatile tick_t end_time; \
expr; \
end_time = tick(); \
fprintf(stderr, "0\t%"PRItick"\t%s\n", end_time - start_time, #expr); \
} while (0)
#else /* USE_TICK_T */
#define MEASURE_LINE(expr) expr
#endif /* USE_TICK_T */
#define FL_TEST2(x,f) ((RGENGC_CHECK_MODE && SPECIAL_CONST_P(x)) ? (rb_bug("FL_TEST2: SPECIAL_CONST (%p)", (void *)(x)), 0) : FL_TEST_RAW((x),(f)) != 0)
#define FL_SET2(x,f) do {if (RGENGC_CHECK_MODE && SPECIAL_CONST_P(x)) rb_bug("FL_SET2: SPECIAL_CONST"); RBASIC(x)->flags |= (f);} while (0)
#define FL_UNSET2(x,f) do {if (RGENGC_CHECK_MODE && SPECIAL_CONST_P(x)) rb_bug("FL_UNSET2: SPECIAL_CONST"); RBASIC(x)->flags &= ~(f);} while (0)
#define RVALUE_MARK_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(obj), (obj))
#define RVALUE_PAGE_MARKED(page, obj) MARKED_IN_BITMAP((page)->mark_bits, (obj))
#if USE_RGENGC
#define RVALUE_WB_UNPROTECTED_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), (obj))
#define RVALUE_UNCOLLECTIBLE_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), (obj))
#define RVALUE_MARKING_BITMAP(obj) MARKED_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), (obj))
#define RVALUE_PAGE_WB_UNPROTECTED(apge, obj) MARKED_IN_BITMAP((page)->wb_unprotected_bits, (obj))
#define RVALUE_PAGE_UNCOLLECTIBLE(page, obj) MARKED_IN_BITMAP((page)->uncollectible_bits, (obj))
#define RVALUE_PAGE_MARKING(page, obj) MARKED_IN_BITMAP((page)->marking_bits, (obj))
#define RVALUE_OLD_AGE 3
#define RVALUE_AGE_SHIFT 5 /* FL_PROMOTED0 bit */
static int rgengc_remembered(rb_objspace_t *objspace, VALUE obj);
static int rgengc_remember(rb_objspace_t *objspace, VALUE obj);
static void rgengc_mark_and_rememberset_clear(rb_objspace_t *objspace, rb_heap_t *heap);
static void rgengc_rememberset_mark(rb_objspace_t *objspace, rb_heap_t *heap);
static inline int
RVALUE_FLAGS_AGE(VALUE flags)
{
return (int)((flags & (FL_PROMOTED0 | FL_PROMOTED1)) >> RVALUE_AGE_SHIFT);
}
#endif /* USE_RGENGC */
#if RGENGC_CHECK_MODE == 0
static inline VALUE
check_rvalue_consistency(const VALUE obj)
{
return obj;
}
#else
static VALUE
check_rvalue_consistency(const VALUE obj)
{
rb_objspace_t *objspace = &rb_objspace;
if (SPECIAL_CONST_P(obj)) {
rb_bug("check_rvalue_consistency: %p is a special const.", (void *)obj);
}
else if (!is_pointer_to_heap(objspace, (void *)obj)) {
rb_bug("check_rvalue_consistency: %p is not a Ruby object.", (void *)obj);
}
else {
const int wb_unprotected_bit = RVALUE_WB_UNPROTECTED_BITMAP(obj) != 0;
const int uncollectible_bit = RVALUE_UNCOLLECTIBLE_BITMAP(obj) != 0;
const int mark_bit = RVALUE_MARK_BITMAP(obj) != 0;
const int marking_bit = RVALUE_MARKING_BITMAP(obj) != 0, remembered_bit = marking_bit;
const int age = RVALUE_FLAGS_AGE(RBASIC(obj)->flags);
if (BUILTIN_TYPE(obj) == T_NONE) rb_bug("check_rvalue_consistency: %s is T_NONE", obj_info(obj));
if (BUILTIN_TYPE(obj) == T_ZOMBIE) rb_bug("check_rvalue_consistency: %s is T_ZOMBIE", obj_info(obj));
obj_memsize_of((VALUE)obj, FALSE);
/* check generation
*
* OLD == age == 3 && old-bitmap && mark-bit (except incremental marking)
*/
if (age > 0 && wb_unprotected_bit) {
rb_bug("check_rvalue_consistency: %s is not WB protected, but age is %d > 0.", obj_info(obj), age);
}
if (!is_marking(objspace) && uncollectible_bit && !mark_bit) {
rb_bug("check_rvalue_consistency: %s is uncollectible, but is not marked while !gc.", obj_info(obj));
}
if (!is_full_marking(objspace)) {
if (uncollectible_bit && age != RVALUE_OLD_AGE && !wb_unprotected_bit) {
rb_bug("check_rvalue_consistency: %s is uncollectible, but not old (age: %d) and not WB unprotected.", obj_info(obj), age);
}
if (remembered_bit && age != RVALUE_OLD_AGE) {
rb_bug("check_rvalue_consistency: %s is rememberd, but not old (age: %d).", obj_info(obj), age);
}
}
/*
* check coloring
*
* marking:false marking:true
* marked:false white *invalid*
* marked:true black grey
*/
if (is_incremental_marking(objspace) && marking_bit) {
if (!is_marking(objspace) && !mark_bit) rb_bug("check_rvalue_consistency: %s is marking, but not marked.", obj_info(obj));
}
}
return obj;
}
#endif
static inline int
RVALUE_MARKED(VALUE obj)
{
check_rvalue_consistency(obj);
return RVALUE_MARK_BITMAP(obj) != 0;
}
#if USE_RGENGC
static inline int
RVALUE_WB_UNPROTECTED(VALUE obj)
{
check_rvalue_consistency(obj);
return RVALUE_WB_UNPROTECTED_BITMAP(obj) != 0;
}
static inline int
RVALUE_MARKING(VALUE obj)
{
check_rvalue_consistency(obj);
return RVALUE_MARKING_BITMAP(obj) != 0;
}
static inline int
RVALUE_REMEMBERED(VALUE obj)
{
check_rvalue_consistency(obj);
return RVALUE_MARKING_BITMAP(obj) != 0;
}
static inline int
RVALUE_UNCOLLECTIBLE(VALUE obj)
{
check_rvalue_consistency(obj);
return RVALUE_UNCOLLECTIBLE_BITMAP(obj) != 0;
}
static inline int
RVALUE_OLD_P_RAW(VALUE obj)
{
const VALUE promoted = FL_PROMOTED0 | FL_PROMOTED1;
return (RBASIC(obj)->flags & promoted) == promoted;
}
static inline int
RVALUE_OLD_P(VALUE obj)
{
check_rvalue_consistency(obj);
return RVALUE_OLD_P_RAW(obj);
}
#if RGENGC_CHECK_MODE || GC_DEBUG
static inline int
RVALUE_AGE(VALUE obj)
{
check_rvalue_consistency(obj);
return RVALUE_FLAGS_AGE(RBASIC(obj)->flags);
}
#endif
static inline void
RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, struct heap_page *page, VALUE obj)
{
MARK_IN_BITMAP(&page->uncollectible_bits[0], obj);
objspace->rgengc.old_objects++;
#if RGENGC_PROFILE >= 2
objspace->profile.total_promoted_count++;
objspace->profile.promoted_types[BUILTIN_TYPE(obj)]++;
#endif
}
static inline void
RVALUE_OLD_UNCOLLECTIBLE_SET(rb_objspace_t *objspace, VALUE obj)
{
RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(objspace, GET_HEAP_PAGE(obj), obj);
}
static inline VALUE
RVALUE_FLAGS_AGE_SET(VALUE flags, int age)
{
flags &= ~(FL_PROMOTED0 | FL_PROMOTED1);
flags |= (age << RVALUE_AGE_SHIFT);
return flags;
}
/* set age to age+1 */
static inline void
RVALUE_AGE_INC(rb_objspace_t *objspace, VALUE obj)
{
VALUE flags = RBASIC(obj)->flags;
int age = RVALUE_FLAGS_AGE(flags);
if (RGENGC_CHECK_MODE && age == RVALUE_OLD_AGE) {
rb_bug("RVALUE_AGE_INC: can not increment age of OLD object %s.", obj_info(obj));
}
age++;
RBASIC(obj)->flags = RVALUE_FLAGS_AGE_SET(flags, age);
if (age == RVALUE_OLD_AGE) {
RVALUE_OLD_UNCOLLECTIBLE_SET(objspace, obj);
}
check_rvalue_consistency(obj);
}
/* set age to RVALUE_OLD_AGE */
static inline void
RVALUE_AGE_SET_OLD(rb_objspace_t *objspace, VALUE obj)
{
check_rvalue_consistency(obj);
if (RGENGC_CHECK_MODE) assert(!RVALUE_OLD_P(obj));
RBASIC(obj)->flags = RVALUE_FLAGS_AGE_SET(RBASIC(obj)->flags, RVALUE_OLD_AGE);
RVALUE_OLD_UNCOLLECTIBLE_SET(objspace, obj);
check_rvalue_consistency(obj);
}
/* set age to RVALUE_OLD_AGE - 1 */
static inline void
RVALUE_AGE_SET_CANDIDATE(rb_objspace_t *objspace, VALUE obj)
{
check_rvalue_consistency(obj);
if (RGENGC_CHECK_MODE) assert(!RVALUE_OLD_P(obj));
RBASIC(obj)->flags = RVALUE_FLAGS_AGE_SET(RBASIC(obj)->flags, RVALUE_OLD_AGE - 1);
check_rvalue_consistency(obj);
}
static inline void
RVALUE_DEMOTE_RAW(rb_objspace_t *objspace, VALUE obj)
{
RBASIC(obj)->flags = RVALUE_FLAGS_AGE_SET(RBASIC(obj)->flags, 0);
CLEAR_IN_BITMAP(GET_HEAP_UNCOLLECTIBLE_BITS(obj), obj);
}
static inline void
RVALUE_DEMOTE(rb_objspace_t *objspace, VALUE obj)
{
check_rvalue_consistency(obj);
if (RGENGC_CHECK_MODE) assert(RVALUE_OLD_P(obj));
if (!is_incremental_marking(objspace) && RVALUE_REMEMBERED(obj)) {
CLEAR_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), obj);
}
RVALUE_DEMOTE_RAW(objspace, obj);
if (RVALUE_MARKED(obj)) {
objspace->rgengc.old_objects--;
}
check_rvalue_consistency(obj);
}
static inline void
RVALUE_AGE_RESET_RAW(VALUE obj)
{
RBASIC(obj)->flags = RVALUE_FLAGS_AGE_SET(RBASIC(obj)->flags, 0);
}
static inline void
RVALUE_AGE_RESET(VALUE obj)
{
check_rvalue_consistency(obj);
if (RGENGC_CHECK_MODE) assert(!RVALUE_OLD_P(obj));
RVALUE_AGE_RESET_RAW(obj);
check_rvalue_consistency(obj);
}
static inline int
RVALUE_BLACK_P(VALUE obj)
{
return RVALUE_MARKED(obj) && !RVALUE_MARKING(obj);
}
#if 0
static inline int
RVALUE_GREY_P(VALUE obj)
{
return RVALUE_MARKED(obj) && RVALUE_MARKING(obj);
}
#endif
static inline int
RVALUE_WHITE_P(VALUE obj)
{
return RVALUE_MARKED(obj) == FALSE;
}
#endif /* USE_RGENGC */
/*
--------------------------- ObjectSpace -----------------------------
*/
#if defined(ENABLE_VM_OBJSPACE) && ENABLE_VM_OBJSPACE
rb_objspace_t *
rb_objspace_alloc(void)
{
rb_objspace_t *objspace = calloc(1, sizeof(rb_objspace_t));
malloc_limit = gc_params.malloc_limit_min;
return objspace;
}
#endif
#if defined(ENABLE_VM_OBJSPACE) && ENABLE_VM_OBJSPACE
static void free_stack_chunks(mark_stack_t *);
static void heap_page_free(rb_objspace_t *objspace, struct heap_page *page);
void
rb_objspace_free(rb_objspace_t *objspace)
{
if (is_lazy_sweeping(heap_eden))
rb_bug("lazy sweeping underway when freeing object space");
if (objspace->profile.records) {
free(objspace->profile.records);
objspace->profile.records = 0;
}
if (global_list) {
struct gc_list *list, *next;
for (list = global_list; list; list = next) {
next = list->next;
xfree(list);
}
}
if (heap_pages_sorted) {
size_t i;
for (i = 0; i < heap_allocated_pages; ++i) {
heap_page_free(objspace, heap_pages_sorted[i]);
}
free(heap_pages_sorted);
heap_allocated_pages = 0;
heap_pages_sorted_length = 0;
heap_pages_lomem = 0;
heap_pages_himem = 0;
objspace->eden_heap.page_length = 0;
objspace->eden_heap.total_slots = 0;
objspace->eden_heap.pages = NULL;
}
free_stack_chunks(&objspace->mark_stack);
free(objspace);
}
#endif
static void
heap_pages_expand_sorted(rb_objspace_t *objspace)
{
size_t next_length = heap_allocatable_pages;
next_length += heap_eden->page_length;
next_length += heap_tomb->page_length;
if (next_length > heap_pages_sorted_length) {
struct heap_page **sorted;
size_t size = next_length * sizeof(struct heap_page *);
gc_report(3, objspace, "heap_pages_expand_sorted: next_length: %d, size: %d\n", (int)next_length, (int)size);
if (heap_pages_sorted_length > 0) {
sorted = (struct heap_page **)realloc(heap_pages_sorted, size);
if (sorted) heap_pages_sorted = sorted;
}
else {
sorted = heap_pages_sorted = (struct heap_page **)malloc(size);
}
if (sorted == 0) {
rb_memerror();
}
heap_pages_sorted_length = next_length;
}
}
static inline void
heap_page_add_freeobj(rb_objspace_t *objspace, struct heap_page *page, VALUE obj)
{
RVALUE *p = (RVALUE *)obj;
p->as.free.flags = 0;
p->as.free.next = page->freelist;
page->freelist = p;
if (RGENGC_CHECK_MODE && !is_pointer_to_heap(objspace, p)) {
rb_bug("heap_page_add_freeobj: %p is not rvalue.", p);
}
gc_report(3, objspace, "heap_page_add_freeobj: add %p to freelist\n", (void *)obj);
}
static inline void
heap_add_freepage(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
{
if (page->freelist) {
page->free_next = heap->free_pages;
heap->free_pages = page;
}
}
#if GC_ENABLE_INCREMENTAL_MARK
static inline int
heap_add_poolpage(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
{
if (page->freelist) {
page->free_next = heap->pooled_pages;
heap->pooled_pages = page;
objspace->rincgc.pooled_slots += page->free_slots;
return TRUE;
}
else {
return FALSE;
}
}
#endif
static void
heap_unlink_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
{
if (page->prev) page->prev->next = page->next;
if (page->next) page->next->prev = page->prev;
if (heap->pages == page) heap->pages = page->next;
page->prev = NULL;
page->next = NULL;
page->heap = NULL;
heap->page_length--;
heap->total_slots -= page->total_slots;
}
static void
heap_page_free(rb_objspace_t *objspace, struct heap_page *page)
{
heap_allocated_pages--;
objspace->profile.total_freed_pages++;
aligned_free(page->body);
free(page);
}
static void
heap_pages_free_unused_pages(rb_objspace_t *objspace)
{
size_t i, j;
if (heap_tomb->pages && heap_pages_swept_slots > heap_pages_max_free_slots) {
for (i = j = 1; j < heap_allocated_pages; i++) {
struct heap_page *page = heap_pages_sorted[i];
if (page->heap == heap_tomb && page->free_slots == page->total_slots) {
if (heap_pages_swept_slots - page->total_slots > heap_pages_max_free_slots) {
if (0) fprintf(stderr, "heap_pages_free_unused_pages: %d free page %p, heap_pages_swept_slots: %d, heap_pages_max_free_slots: %d\n",
(int)i, page, (int)heap_pages_swept_slots, (int)heap_pages_max_free_slots);
heap_pages_swept_slots -= page->total_slots;
heap_unlink_page(objspace, heap_tomb, page);
heap_page_free(objspace, page);
continue;
}
else if (i == j) {
return; /* no need to check rest pages */
}
}
if (i != j) {
heap_pages_sorted[j] = page;
}
j++;
}
if (RGENGC_CHECK_MODE) assert(j == heap_allocated_pages);
}
}
static struct heap_page *
heap_page_allocate(rb_objspace_t *objspace)
{
RVALUE *start, *end, *p;
struct heap_page *page;
struct heap_page_body *page_body = 0;
size_t hi, lo, mid;
int limit = HEAP_OBJ_LIMIT;
/* assign heap_page body (contains heap_page_header and RVALUEs) */
page_body = (struct heap_page_body *)aligned_malloc(HEAP_ALIGN, HEAP_SIZE);
if (page_body == 0) {
rb_memerror();
}
/* assign heap_page entry */
page = (struct heap_page *)calloc(1, sizeof(struct heap_page));
if (page == 0) {
aligned_free(page_body);
rb_memerror();
}
page->body = page_body;
/* setup heap_pages_sorted */
lo = 0;
hi = heap_allocated_pages;
while (lo < hi) {
struct heap_page *mid_page;
mid = (lo + hi) / 2;
mid_page = heap_pages_sorted[mid];
if (mid_page->body < page_body) {
lo = mid + 1;
}
else if (mid_page->body > page_body) {
hi = mid;
}
else {
rb_bug("same heap page is allocated: %p at %"PRIuVALUE, (void *)page_body, (VALUE)mid);
}
}
if (hi < heap_allocated_pages) {
MEMMOVE(&heap_pages_sorted[hi+1], &heap_pages_sorted[hi], struct heap_page_header*, heap_allocated_pages - hi);
}
heap_pages_sorted[hi] = page;
heap_allocated_pages++;
objspace->profile.total_allocated_pages++;
if (RGENGC_CHECK_MODE) assert(heap_allocated_pages <= heap_pages_sorted_length);
/* adjust obj_limit (object number available in this page) */
start = (RVALUE*)((VALUE)page_body + sizeof(struct heap_page_header));
if ((VALUE)start % sizeof(RVALUE) != 0) {
int delta = (int)(sizeof(RVALUE) - ((VALUE)start % sizeof(RVALUE)));
start = (RVALUE*)((VALUE)start + delta);
limit = (HEAP_SIZE - (int)((VALUE)start - (VALUE)page_body))/(int)sizeof(RVALUE);
}
end = start + limit;
if (heap_pages_lomem == 0 || heap_pages_lomem > start) heap_pages_lomem = start;
if (heap_pages_himem < end) heap_pages_himem = end;
page->start = start;
page->total_slots = limit;
page_body->header.page = page;
for (p = start; p != end; p++) {
gc_report(3, objspace, "assign_heap_page: %p is added to freelist\n", p);
heap_page_add_freeobj(objspace, page, (VALUE)p);
}
page->free_slots = limit;
return page;
}
static struct heap_page *
heap_page_resurrect(rb_objspace_t *objspace)
{
struct heap_page *page;
if ((page = heap_tomb->pages) != NULL) {
heap_unlink_page(objspace, heap_tomb, page);
return page;
}
return NULL;
}
static struct heap_page *
heap_page_create(rb_objspace_t *objspace)
{
struct heap_page *page = heap_page_resurrect(objspace);
const char *method = "recycle";
if (page == NULL) {
page = heap_page_allocate(objspace);
method = "allocate";
}
if (0) fprintf(stderr, "heap_page_create: %s - %p, heap_allocated_pages: %d, heap_allocated_pages: %d, tomb->page_length: %d\n",
method, page, (int)heap_pages_sorted_length, (int)heap_allocated_pages, (int)heap_tomb->page_length);
return page;
}
static void
heap_add_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
{
page->heap = heap;
page->next = heap->pages;
if (heap->pages) heap->pages->prev = page;
heap->pages = page;
heap->page_length++;
heap->total_slots += page->total_slots;
}
static void
heap_assign_page(rb_objspace_t *objspace, rb_heap_t *heap)
{
struct heap_page *page = heap_page_create(objspace);
heap_add_page(objspace, heap, page);
heap_add_freepage(objspace, heap, page);
}
static void
heap_add_pages(rb_objspace_t *objspace, rb_heap_t *heap, size_t add)
{
size_t i;
heap_allocatable_pages = add;
heap_pages_expand_sorted(objspace);
for (i = 0; i < add; i++) {
heap_assign_page(objspace, heap);
}
heap_allocatable_pages = 0;
}
static size_t
heap_extend_pages(rb_objspace_t *objspace)
{
size_t used = heap_allocated_pages - heap_tomb->page_length;
size_t next_used_limit = (size_t)(used * gc_params.growth_factor);
if (gc_params.growth_max_slots > 0) {
size_t max_used_limit = (size_t)(used + gc_params.growth_max_slots/HEAP_OBJ_LIMIT);
if (next_used_limit > max_used_limit) next_used_limit = max_used_limit;
}
return next_used_limit - used;
}
static void
heap_set_increment(rb_objspace_t *objspace, size_t additional_pages)
{
size_t used = heap_eden->page_length;
size_t next_used_limit = used + additional_pages;
if (next_used_limit == heap_allocated_pages) next_used_limit++;
heap_allocatable_pages = next_used_limit - used;
heap_pages_expand_sorted(objspace);
gc_report(1, objspace, "heap_set_increment: heap_allocatable_pages is %d\n", (int)heap_allocatable_pages);
}
static int
heap_increment(rb_objspace_t *objspace, rb_heap_t *heap)
{
if (heap_allocatable_pages > 0) {
gc_report(1, objspace, "heap_increment: heap_pages_sorted_length: %d, heap_pages_inc: %d, heap->page_length: %d\n",
(int)heap_pages_sorted_length, (int)heap_allocatable_pages, (int)heap->page_length);
heap_allocatable_pages--;
heap_assign_page(objspace, heap);
return TRUE;
}
return FALSE;
}
static void
heap_prepare(rb_objspace_t *objspace, rb_heap_t *heap)
{
if (RGENGC_CHECK_MODE) assert(heap->free_pages == NULL);
#if GC_ENABLE_LAZY_SWEEP
if (is_lazy_sweeping(heap)) {
gc_sweep_continue(objspace, heap);
}
#endif
#if GC_ENABLE_INCREMENTAL_MARK
else if (is_incremental_marking(objspace)) {
gc_marks_continue(objspace, heap);
}
#endif
if (heap->free_pages == NULL &&
(will_be_incremental_marking(objspace) || heap_increment(objspace, heap) == FALSE) &&
gc_start(objspace, FALSE, FALSE, FALSE, GPR_FLAG_NEWOBJ) == FALSE) {
rb_memerror();
}
}
static RVALUE *
heap_get_freeobj_from_next_freepage(rb_objspace_t *objspace, rb_heap_t *heap)
{
struct heap_page *page;
RVALUE *p;
while (UNLIKELY(heap->free_pages == NULL)) {
heap_prepare(objspace, heap);
}
page = heap->free_pages;
heap->free_pages = page->free_next;
heap->using_page = page;
if (RGENGC_CHECK_MODE) assert(page->free_slots != 0);
p = page->freelist;
page->freelist = NULL;
page->free_slots = 0;
return p;
}
static inline VALUE
heap_get_freeobj(rb_objspace_t *objspace, rb_heap_t *heap)
{
RVALUE *p = heap->freelist;
while (1) {
if (LIKELY(p != NULL)) {
heap->freelist = p->as.free.next;
return (VALUE)p;
}
else {
p = heap_get_freeobj_from_next_freepage(objspace, heap);
}
}
}
void
rb_objspace_set_event_hook(const rb_event_flag_t event)
{
rb_objspace_t *objspace = &rb_objspace;
objspace->hook_events = event & RUBY_INTERNAL_EVENT_OBJSPACE_MASK;
}
static void
gc_event_hook_body(rb_objspace_t *objspace, const rb_event_flag_t event, VALUE data)
{
rb_thread_t *th = GET_THREAD();
EXEC_EVENT_HOOK(th, event, th->cfp->self, 0, 0, data);
}
#define gc_event_hook(objspace, event, data) do { \
if (UNLIKELY((objspace)->hook_events & (event))) { \
gc_event_hook_body((objspace), (event), (data)); \
} \
} while (0)
static VALUE
newobj_of(VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3)
{
rb_objspace_t *objspace = &rb_objspace;
VALUE obj;
if (UNLIKELY(during_gc || ruby_gc_stressful)) {
if (during_gc) {
dont_gc = 1;
during_gc = 0;
rb_bug("object allocation during garbage collection phase");
}
if (ruby_gc_stressful) {
if (!garbage_collect(objspace, FALSE, FALSE, FALSE, GPR_FLAG_NEWOBJ)) {
rb_memerror();
}
}
}
obj = heap_get_freeobj(objspace, heap_eden);
if (RGENGC_CHECK_MODE > 0) assert(BUILTIN_TYPE(obj) == T_NONE);
/* OBJSETUP */
RBASIC(obj)->flags = flags & ~FL_WB_PROTECTED;
RBASIC_SET_CLASS_RAW(obj, klass);
if (rb_safe_level() >= 3) FL_SET((obj), FL_TAINT);
RANY(obj)->as.values.v1 = v1;
RANY(obj)->as.values.v2 = v2;
RANY(obj)->as.values.v3 = v3;
#if RGENGC_CHECK_MODE
assert(RVALUE_MARKED(obj) == FALSE);
assert(RVALUE_MARKING(obj) == FALSE);
assert(RVALUE_OLD_P(obj) == FALSE);
assert(RVALUE_WB_UNPROTECTED(obj) == FALSE);
if (flags & FL_PROMOTED1) {
if (RVALUE_AGE(obj) != 2) rb_bug("newobj: %s of age (%d) != 2.", obj_info(obj), RVALUE_AGE(obj));
}
else {
if (RVALUE_AGE(obj) > 0) rb_bug("newobj: %s of age (%d) > 0.", obj_info(obj), RVALUE_AGE(obj));
}
if (rgengc_remembered(objspace, (VALUE)obj)) rb_bug("newobj: %s is remembered.", obj_info(obj));
#endif
#if USE_RGENGC
if ((flags & FL_WB_PROTECTED) == 0) {
MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
}
#endif
#if RGENGC_PROFILE
if (flags & FL_WB_PROTECTED) {
objspace->profile.total_generated_normal_object_count++;
#if RGENGC_PROFILE >= 2
objspace->profile.generated_normal_object_count_types[BUILTIN_TYPE(obj)]++;
#endif
}
else {
objspace->profile.total_generated_shady_object_count++;
#if RGENGC_PROFILE >= 2
objspace->profile.generated_shady_object_count_types[BUILTIN_TYPE(obj)]++;
#endif
}
#endif
#if GC_DEBUG
RANY(obj)->file = rb_sourcefile();
RANY(obj)->line = rb_sourceline();
assert(!SPECIAL_CONST_P(obj)); /* check alignment */
#endif
objspace->total_allocated_objects++;
gc_event_hook(objspace, RUBY_INTERNAL_EVENT_NEWOBJ, obj);
gc_report(5, objspace, "newobj: %s\n", obj_info(obj));
#if RGENGC_OLD_NEWOBJ_CHECK > 0
{
static int newobj_cnt = RGENGC_OLD_NEWOBJ_CHECK;
if (!is_incremental_marking(objspace) &&
flags & FL_WB_PROTECTED && /* do not promote WB unprotected objects */
! RB_TYPE_P(obj, T_ARRAY)) { /* array.c assumes that allocated objects are new */
if (--newobj_cnt == 0) {
newobj_cnt = RGENGC_OLD_NEWOBJ_CHECK;
gc_mark_set(objspace, obj);
RVALUE_AGE_SET_OLD(objspace, obj);
rb_gc_writebarrier_remember(obj);
}
}
}
#endif
check_rvalue_consistency(obj);
return obj;
}
VALUE
rb_newobj(void)
{
return newobj_of(0, T_NONE, 0, 0, 0);
}
VALUE
rb_newobj_of(VALUE klass, VALUE flags)
{
return newobj_of(klass, flags, 0, 0, 0);
}
NODE*
rb_node_newnode(enum node_type type, VALUE a0, VALUE a1, VALUE a2)
{
VALUE flags = 0;
NODE *n = (NODE *)newobj_of(0, T_NODE | flags, a0, a1, a2);
nd_set_type(n, type);
return n;
}
#undef rb_imemo_new
VALUE
rb_imemo_new(enum imemo_type type, VALUE v1, VALUE v2, VALUE v3, VALUE v0)
{
VALUE flags = T_IMEMO | (type << FL_USHIFT) | FL_WB_PROTECTED;
return newobj_of(v0, flags, v1, v2, v3);
}
#if IMEMO_DEBUG
VALUE
rb_imemo_new_debug(enum imemo_type type, VALUE v1, VALUE v2, VALUE v3, VALUE v0, const char *file, int line)
{
VALUE memo = rb_imemo_new(type, v1, v2, v3, v0);
fprintf(stderr, "memo %p (type: %d) @ %s:%d\n", memo, imemo_type(memo), file, line);
return memo;
}
#endif
VALUE
rb_data_object_alloc(VALUE klass, void *datap, RUBY_DATA_FUNC dmark, RUBY_DATA_FUNC dfree)
{
if (klass) Check_Type(klass, T_CLASS);
return newobj_of(klass, T_DATA, (VALUE)dmark, (VALUE)dfree, (VALUE)datap);
}
VALUE
rb_data_typed_object_alloc(VALUE klass, void *datap, const rb_data_type_t *type)
{
if (klass) Check_Type(klass, T_CLASS);
return newobj_of(klass, T_DATA | (type->flags & ~T_MASK), (VALUE)type, (VALUE)1, (VALUE)datap);
}
size_t
rb_objspace_data_type_memsize(VALUE obj)
{
if (RTYPEDDATA_P(obj) && RTYPEDDATA_TYPE(obj)->function.dsize) {
return RTYPEDDATA_TYPE(obj)->function.dsize(RTYPEDDATA_DATA(obj));
}
else {
return 0;
}
}
const char *
rb_objspace_data_type_name(VALUE obj)
{
if (RTYPEDDATA_P(obj)) {
return RTYPEDDATA_TYPE(obj)->wrap_struct_name;
}
else {
return 0;
}
}
static inline int
is_pointer_to_heap(rb_objspace_t *objspace, void *ptr)
{
register RVALUE *p = RANY(ptr);
register struct heap_page *page;
register size_t hi, lo, mid;
if (p < heap_pages_lomem || p > heap_pages_himem) return FALSE;
if ((VALUE)p % sizeof(RVALUE) != 0) return FALSE;
/* check if p looks like a pointer using bsearch*/
lo = 0;
hi = heap_allocated_pages;
while (lo < hi) {
mid = (lo + hi) / 2;
page = heap_pages_sorted[mid];
if (page->start <= p) {
if (p < page->start + page->total_slots) {
return TRUE;
}
lo = mid + 1;
}
else {
hi = mid;
}
}
return FALSE;
}
static int
free_method_entry_i(st_data_t key, st_data_t value, st_data_t data)
{
rb_method_entry_t *me = (rb_method_entry_t *)value;
if (!me->mark) {
rb_free_method_entry(me);
}
return ST_CONTINUE;
}
static void
rb_free_m_tbl(st_table *tbl)
{
if (tbl) {
st_foreach(tbl, free_method_entry_i, 0);
st_free_table(tbl);
}
}
static int
free_const_entry_i(st_data_t key, st_data_t value, st_data_t data)
{
rb_const_entry_t *ce = (rb_const_entry_t *)value;
xfree(ce);
return ST_CONTINUE;
}
void
rb_free_const_table(st_table *tbl)
{
st_foreach(tbl, free_const_entry_i, 0);
st_free_table(tbl);
}
static inline void
make_zombie(rb_objspace_t *objspace, VALUE obj, void (*dfree)(void *), void *data)
{
struct RZombie *zombie = RZOMBIE(obj);
zombie->basic.flags = T_ZOMBIE;
zombie->dfree = dfree;
zombie->data = data;
zombie->next = heap_pages_deferred_final;
heap_pages_deferred_final = (VALUE)zombie;
}
static inline void
make_io_zombie(rb_objspace_t *objspace, VALUE obj)
{
rb_io_t *fptr = RANY(obj)->as.file.fptr;
make_zombie(objspace, obj, (void (*)(void*))rb_io_fptr_finalize, fptr);
}
static int
obj_free(rb_objspace_t *objspace, VALUE obj)
{
gc_event_hook(objspace, RUBY_INTERNAL_EVENT_FREEOBJ, obj);
switch (BUILTIN_TYPE(obj)) {
case T_NIL:
case T_FIXNUM:
case T_TRUE:
case T_FALSE:
rb_bug("obj_free() called for broken object");
break;
}
if (FL_TEST(obj, FL_EXIVAR)) {
rb_free_generic_ivar((VALUE)obj);
FL_UNSET(obj, FL_EXIVAR);
}
#if USE_RGENGC
if (RVALUE_WB_UNPROTECTED(obj)) CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
#if RGENGC_CHECK_MODE
#define CHECK(x) if (x(obj) != FALSE) rb_bug("obj_free: " #x "(%s) != FALSE", obj_info(obj))
CHECK(RVALUE_WB_UNPROTECTED);
CHECK(RVALUE_MARKED);
CHECK(RVALUE_MARKING);
CHECK(RVALUE_UNCOLLECTIBLE);
#undef CHECK
#endif
#endif
switch (BUILTIN_TYPE(obj)) {
case T_OBJECT:
if (!(RANY(obj)->as.basic.flags & ROBJECT_EMBED) &&
RANY(obj)->as.object.as.heap.ivptr) {
xfree(RANY(obj)->as.object.as.heap.ivptr);
}
break;
case T_MODULE:
case T_CLASS:
rb_free_m_tbl(RCLASS_M_TBL(obj));
if (RCLASS_IV_TBL(obj)) {
st_free_table(RCLASS_IV_TBL(obj));
}
if (RCLASS_CONST_TBL(obj)) {
rb_free_const_table(RCLASS_CONST_TBL(obj));
}
if (RCLASS_IV_INDEX_TBL(obj)) {
st_free_table(RCLASS_IV_INDEX_TBL(obj));
}
if (RCLASS_EXT(obj)->subclasses) {
if (BUILTIN_TYPE(obj) == T_MODULE) {
rb_class_detach_module_subclasses(obj);
}
else {
rb_class_detach_subclasses(obj);
}
RCLASS_EXT(obj)->subclasses = NULL;
}
rb_class_remove_from_module_subclasses(obj);
rb_class_remove_from_super_subclasses(obj);
if (RANY(obj)->as.klass.ptr)
xfree(RANY(obj)->as.klass.ptr);
RANY(obj)->as.klass.ptr = NULL;
break;
case T_STRING:
rb_str_free(obj);
break;
case T_ARRAY:
rb_ary_free(obj);
break;
case T_HASH:
if (RANY(obj)->as.hash.ntbl) {
st_free_table(RANY(obj)->as.hash.ntbl);
}
break;
case T_REGEXP:
if (RANY(obj)->as.regexp.ptr) {
onig_free(RANY(obj)->as.regexp.ptr);
}
break;
case T_DATA:
if (DATA_PTR(obj)) {
int free_immediately = FALSE;
void (*dfree)(void *);
void *data = DATA_PTR(obj);
if (RTYPEDDATA_P(obj)) {
free_immediately = (RANY(obj)->as.typeddata.type->flags & RUBY_TYPED_FREE_IMMEDIATELY) != 0;
dfree = RANY(obj)->as.typeddata.type->function.dfree;
if (0 && free_immediately == 0) {
/* to expose non-free-immediate T_DATA */
fprintf(stderr, "not immediate -> %s\n", RANY(obj)->as.typeddata.type->wrap_struct_name);
}
}
else {
dfree = RANY(obj)->as.data.dfree;
}
if (dfree) {
if (dfree == RUBY_DEFAULT_FREE) {
xfree(data);
}
else if (free_immediately) {
(*dfree)(data);
}
else {
make_zombie(objspace, obj, dfree, data);
return 1;
}
}
}
break;
case T_MATCH:
if (RANY(obj)->as.match.rmatch) {
struct rmatch *rm = RANY(obj)->as.match.rmatch;
onig_region_free(&rm->regs, 0);
if (rm->char_offset)
xfree(rm->char_offset);
xfree(rm);
}
break;
case T_FILE:
if (RANY(obj)->as.file.fptr) {
make_io_zombie(objspace, obj);
return 1;
}
break;
case T_RATIONAL:
case T_COMPLEX:
case T_IMEMO:
break;
case T_ICLASS:
/* Basically , T_ICLASS shares table with the module */
if (FL_TEST(obj, RICLASS_IS_ORIGIN)) {
rb_free_m_tbl(RCLASS_M_TBL(obj));
}
if (RCLASS_EXT(obj)->subclasses) {
rb_class_detach_subclasses(obj);
RCLASS_EXT(obj)->subclasses = NULL;
}
rb_class_remove_from_module_subclasses(obj);
rb_class_remove_from_super_subclasses(obj);
xfree(RANY(obj)->as.klass.ptr);
RANY(obj)->as.klass.ptr = NULL;
break;
case T_FLOAT:
break;
case T_BIGNUM:
if (!(RBASIC(obj)->flags & BIGNUM_EMBED_FLAG) && BIGNUM_DIGITS(obj)) {
xfree(BIGNUM_DIGITS(obj));
}
break;
case T_NODE:
rb_gc_free_node(obj);
break; /* no need to free iv_tbl */
case T_STRUCT:
if ((RBASIC(obj)->flags & RSTRUCT_EMBED_LEN_MASK) == 0 &&
RANY(obj)->as.rstruct.as.heap.ptr) {
xfree((void *)RANY(obj)->as.rstruct.as.heap.ptr);
}
break;
case T_SYMBOL:
{
rb_gc_free_dsymbol(obj);
}
break;
default:
rb_bug("gc_sweep(): unknown data type 0x%x(%p) 0x%"PRIxVALUE,
BUILTIN_TYPE(obj), (void*)obj, RBASIC(obj)->flags);
}
if (FL_TEST(obj, FL_FINALIZE)) {
make_zombie(objspace, obj, 0, 0);
return 1;
}
else {
return 0;
}
}
void
Init_heap(void)
{
rb_objspace_t *objspace = &rb_objspace;
gc_stress_set(objspace, ruby_initial_gc_stress);
#if RGENGC_ESTIMATE_OLDMALLOC
objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min;
#endif
heap_add_pages(objspace, heap_eden, gc_params.heap_init_slots / HEAP_OBJ_LIMIT);
init_mark_stack(&objspace->mark_stack);
#ifdef USE_SIGALTSTACK
{
/* altstack of another threads are allocated in another place */
rb_thread_t *th = GET_THREAD();
void *tmp = th->altstack;
th->altstack = malloc(rb_sigaltstack_size());
free(tmp); /* free previously allocated area */
}
#endif
objspace->profile.invoke_time = getrusage_time();
finalizer_table = st_init_numtable();
}
typedef int each_obj_callback(void *, void *, size_t, void *);
struct each_obj_args {
each_obj_callback *callback;
void *data;
};
static VALUE
objspace_each_objects(VALUE arg)
{
size_t i;
struct heap_page_body *last_body = 0;
struct heap_page *page;
RVALUE *pstart, *pend;
rb_objspace_t *objspace = &rb_objspace;
struct each_obj_args *args = (struct each_obj_args *)arg;
i = 0;
while (i < heap_allocated_pages) {
while (0 < i && last_body < heap_pages_sorted[i-1]->body) i--;
while (i < heap_allocated_pages && heap_pages_sorted[i]->body <= last_body) i++;
if (heap_allocated_pages <= i) break;
page = heap_pages_sorted[i];
last_body = page->body;
pstart = page->start;
pend = pstart + page->total_slots;
if ((*args->callback)(pstart, pend, sizeof(RVALUE), args->data)) {
break;
}
}
return Qnil;
}
static VALUE
incremental_enable(void)
{
rb_objspace_t *objspace = &rb_objspace;
objspace->flags.dont_incremental = FALSE;
return Qnil;
}
/*
* rb_objspace_each_objects() is special C API to walk through
* Ruby object space. This C API is too difficult to use it.
* To be frank, you should not use it. Or you need to read the
* source code of this function and understand what this function does.
*
* 'callback' will be called several times (the number of heap page,
* at current implementation) with:
* vstart: a pointer to the first living object of the heap_page.
* vend: a pointer to next to the valid heap_page area.
* stride: a distance to next VALUE.
*
* If callback() returns non-zero, the iteration will be stopped.
*
* This is a sample callback code to iterate liveness objects:
*
* int
* sample_callback(void *vstart, void *vend, int stride, void *data) {
* VALUE v = (VALUE)vstart;
* for (; v != (VALUE)vend; v += stride) {
* if (RBASIC(v)->flags) { // liveness check
* // do something with live object 'v'
* }
* return 0; // continue to iteration
* }
*
* Note: 'vstart' is not a top of heap_page. This point the first
* living object to grasp at least one object to avoid GC issue.
* This means that you can not walk through all Ruby object page
* including freed object page.
*
* Note: On this implementation, 'stride' is same as sizeof(RVALUE).
* However, there are possibilities to pass variable values with
* 'stride' with some reasons. You must use stride instead of
* use some constant value in the iteration.
*/
void
rb_objspace_each_objects(each_obj_callback *callback, void *data)
{
struct each_obj_args args;
rb_objspace_t *objspace = &rb_objspace;
int prev_dont_incremental = objspace->flags.dont_incremental;
gc_rest(objspace);
objspace->flags.dont_incremental = TRUE;
args.callback = callback;
args.data = data;
if (prev_dont_incremental) {
objspace_each_objects((VALUE)&args);
}
else {
rb_ensure(objspace_each_objects, (VALUE)&args, incremental_enable, Qnil);
}
}
void
rb_objspace_each_objects_without_setup(each_obj_callback *callback, void *data)
{
struct each_obj_args args;
args.callback = callback;
args.data = data;
objspace_each_objects((VALUE)&args);
}
struct os_each_struct {
size_t num;
VALUE of;
};
static int
internal_object_p(VALUE obj)
{
RVALUE *p = (RVALUE *)obj;
if (p->as.basic.flags) {
switch (BUILTIN_TYPE(p)) {
case T_NONE:
case T_IMEMO:
case T_ICLASS:
case T_NODE:
case T_ZOMBIE:
break;
case T_CLASS:
if (FL_TEST(p, FL_SINGLETON))
break;
default:
if (!p->as.basic.klass) break;
return 0;
}
}
return 1;
}
int
rb_objspace_internal_object_p(VALUE obj)
{
return internal_object_p(obj);
}
static int
os_obj_of_i(void *vstart, void *vend, size_t stride, void *data)
{
struct os_each_struct *oes = (struct os_each_struct *)data;
RVALUE *p = (RVALUE *)vstart, *pend = (RVALUE *)vend;
for (; p != pend; p++) {
volatile VALUE v = (VALUE)p;
if (!internal_object_p(v)) {
if (!oes->of || rb_obj_is_kind_of(v, oes->of)) {
rb_yield(v);
oes->num++;
}
}
}
return 0;
}
static VALUE
os_obj_of(VALUE of)
{
struct os_each_struct oes;
oes.num = 0;
oes.of = of;
rb_objspace_each_objects(os_obj_of_i, &oes);
return SIZET2NUM(oes.num);
}
/*
* call-seq:
* ObjectSpace.each_object([module]) {|obj| ... } -> fixnum
* ObjectSpace.each_object([module]) -> an_enumerator
*
* Calls the block once for each living, nonimmediate object in this
* Ruby process. If <i>module</i> is specified, calls the block
* for only those classes or modules that match (or are a subclass of)
* <i>module</i>. Returns the number of objects found. Immediate
* objects (<code>Fixnum</code>s, <code>Symbol</code>s
* <code>true</code>, <code>false</code>, and <code>nil</code>) are
* never returned. In the example below, <code>each_object</code>
* returns both the numbers we defined and several constants defined in
* the <code>Math</code> module.
*
* If no block is given, an enumerator is returned instead.
*
* a = 102.7
* b = 95 # Won't be returned
* c = 12345678987654321
* count = ObjectSpace.each_object(Numeric) {|x| p x }
* puts "Total count: #{count}"
*
* <em>produces:</em>
*
* 12345678987654321
* 102.7
* 2.71828182845905
* 3.14159265358979
* 2.22044604925031e-16
* 1.7976931348623157e+308
* 2.2250738585072e-308
* Total count: 7
*
*/
static VALUE
os_each_obj(int argc, VALUE *argv, VALUE os)
{
VALUE of;
if (argc == 0) {
of = 0;
}
else {
rb_scan_args(argc, argv, "01", &of);
}
RETURN_ENUMERATOR(os, 1, &of);
return os_obj_of(of);
}
/*
* call-seq:
* ObjectSpace.undefine_finalizer(obj)
*
* Removes all finalizers for <i>obj</i>.
*
*/
static VALUE
undefine_final(VALUE os, VALUE obj)
{
return rb_undefine_finalizer(obj);
}
VALUE
rb_undefine_finalizer(VALUE obj)
{
rb_objspace_t *objspace = &rb_objspace;
st_data_t data = obj;
rb_check_frozen(obj);
st_delete(finalizer_table, &data, 0);
FL_UNSET(obj, FL_FINALIZE);
return obj;
}
static void
should_be_callable(VALUE block)
{
if (!rb_obj_respond_to(block, rb_intern("call"), TRUE)) {
rb_raise(rb_eArgError, "wrong type argument %"PRIsVALUE" (should be callable)",
rb_obj_class(block));
}
}
static void
should_be_finalizable(VALUE obj)
{
if (!FL_ABLE(obj)) {
rb_raise(rb_eArgError, "cannot define finalizer for %s",
rb_obj_classname(obj));
}
rb_check_frozen(obj);
}
/*
* call-seq:
* ObjectSpace.define_finalizer(obj, aProc=proc())
*
* Adds <i>aProc</i> as a finalizer, to be called after <i>obj</i>
* was destroyed.
*
*/
static VALUE
define_final(int argc, VALUE *argv, VALUE os)
{
VALUE obj, block;
rb_scan_args(argc, argv, "11", &obj, &block);
should_be_finalizable(obj);
if (argc == 1) {
block = rb_block_proc();
}
else {
should_be_callable(block);
}
return define_final0(obj, block);
}
static VALUE
define_final0(VALUE obj, VALUE block)
{
rb_objspace_t *objspace = &rb_objspace;
VALUE table;
st_data_t data;
RBASIC(obj)->flags |= FL_FINALIZE;
block = rb_ary_new3(2, INT2FIX(rb_safe_level()), block);
OBJ_FREEZE(block);
if (st_lookup(finalizer_table, obj, &data)) {
table = (VALUE)data;
/* avoid duplicate block, table is usually small */
{
const VALUE *ptr = RARRAY_CONST_PTR(table);
long len = RARRAY_LEN(table);
long i;
for (i = 0; i < len; i++, ptr++) {
if (rb_funcall(*ptr, idEq, 1, block)) {
return *ptr;
}
}
}
rb_ary_push(table, block);
}
else {
table = rb_ary_new3(1, block);
RBASIC_CLEAR_CLASS(table);
st_add_direct(finalizer_table, obj, table);
}
return block;
}
VALUE
rb_define_finalizer(VALUE obj, VALUE block)
{
should_be_finalizable(obj);
should_be_callable(block);
return define_final0(obj, block);
}
void
rb_gc_copy_finalizer(VALUE dest, VALUE obj)
{
rb_objspace_t *objspace = &rb_objspace;
VALUE table;
st_data_t data;
if (!FL_TEST(obj, FL_FINALIZE)) return;
if (st_lookup(finalizer_table, obj, &data)) {
table = (VALUE)data;
st_insert(finalizer_table, dest, table);
}
FL_SET(dest, FL_FINALIZE);
}
static VALUE
run_single_final(VALUE arg)
{
VALUE *args = (VALUE *)arg;
rb_eval_cmd(args[0], args[1], (int)args[2]);
return Qnil;
}
static void
run_finalizer(rb_objspace_t *objspace, VALUE obj, VALUE table)
{
long i;
int status;
VALUE args[3];
VALUE objid = nonspecial_obj_id(obj);
if (RARRAY_LEN(table) > 0) {
args[1] = rb_obj_freeze(rb_ary_new3(1, objid));
}
else {
args[1] = 0;
}
args[2] = (VALUE)rb_safe_level();
for (i=0; i<RARRAY_LEN(table); i++) {
VALUE final = RARRAY_AREF(table, i);
args[0] = RARRAY_AREF(final, 1);
args[2] = FIX2INT(RARRAY_AREF(final, 0));
status = 0;
rb_protect(run_single_final, (VALUE)args, &status);
if (status)
rb_set_errinfo(Qnil);
}
}
static void
run_final(rb_objspace_t *objspace, VALUE zombie)
{
st_data_t key, table;
if (RZOMBIE(zombie)->dfree) {
RZOMBIE(zombie)->dfree(RZOMBIE(zombie)->data);
}
key = (st_data_t)zombie;
if (st_delete(finalizer_table, &key, &table)) {
run_finalizer(objspace, zombie, (VALUE)table);
}
}
static void
finalize_list(rb_objspace_t *objspace, VALUE zombie)
{
while (zombie) {
VALUE next_zombie = RZOMBIE(zombie)->next;
struct heap_page *page = GET_HEAP_PAGE(zombie);
run_final(objspace, zombie);
RZOMBIE(zombie)->basic.flags = 0;
heap_pages_final_slots--;
page->final_slots--;
page->free_slots++;
heap_page_add_freeobj(objspace, GET_HEAP_PAGE(zombie), zombie);
heap_pages_swept_slots++;
objspace->profile.total_freed_objects++;
zombie = next_zombie;
}
}
static void
finalize_deferred(rb_objspace_t *objspace)
{
VALUE zombie;
while ((zombie = (VALUE)ATOMIC_PTR_EXCHANGE(heap_pages_deferred_final, 0)) != 0) {
finalize_list(objspace, zombie);
}
}
static void
gc_finalize_deferred(void *dmy)
{
rb_objspace_t *objspace = &rb_objspace;
if (ATOMIC_EXCHANGE(finalizing, 1)) return;
finalize_deferred(objspace);
ATOMIC_SET(finalizing, 0);
}
/* TODO: to keep compatibility, maybe unused. */
void
rb_gc_finalize_deferred(void)
{
gc_finalize_deferred(0);
}
static void
gc_finalize_deferred_register(void)
{
if (rb_postponed_job_register_one(0, gc_finalize_deferred, 0) == 0) {
rb_bug("gc_finalize_deferred_register: can't register finalizer.");
}
}
struct force_finalize_list {
VALUE obj;
VALUE table;
struct force_finalize_list *next;
};
static int
force_chain_object(st_data_t key, st_data_t val, st_data_t arg)
{
struct force_finalize_list **prev = (struct force_finalize_list **)arg;
struct force_finalize_list *curr = ALLOC(struct force_finalize_list);
curr->obj = key;
curr->table = val;
curr->next = *prev;
*prev = curr;
return ST_CONTINUE;
}
void
rb_gc_call_finalizer_at_exit(void)
{
#if RGENGC_CHECK_MODE >= 2
gc_verify_internal_consistency(Qnil);
#endif
rb_objspace_call_finalizer(&rb_objspace);
}
static void
rb_objspace_call_finalizer(rb_objspace_t *objspace)
{
RVALUE *p, *pend;
size_t i;
gc_rest(objspace);
if (ATOMIC_EXCHANGE(finalizing, 1)) return;
/* run finalizers */
finalize_deferred(objspace);
assert(heap_pages_deferred_final == 0);
gc_rest(objspace);
/* prohibit incremental GC */
objspace->flags.dont_incremental = 1;
/* force to run finalizer */
while (finalizer_table->num_entries) {
struct force_finalize_list *list = 0;
st_foreach(finalizer_table, force_chain_object, (st_data_t)&list);
while (list) {
struct force_finalize_list *curr = list;
st_data_t obj = (st_data_t)curr->obj;
run_finalizer(objspace, curr->obj, curr->table);
st_delete(finalizer_table, &obj, 0);
list = curr->next;
xfree(curr);
}
}
/* prohibit GC because force T_DATA finalizers can break an object graph consistency */
dont_gc = 1;
/* running data/file finalizers are part of garbage collection */
gc_enter(objspace, "rb_objspace_call_finalizer");
/* run data/file object's finalizers */
for (i = 0; i < heap_allocated_pages; i++) {
p = heap_pages_sorted[i]->start; pend = p + heap_pages_sorted[i]->total_slots;
while (p < pend) {
switch (BUILTIN_TYPE(p)) {
case T_DATA:
if (!DATA_PTR(p) || !RANY(p)->as.data.dfree) break;
if (rb_obj_is_thread((VALUE)p)) break;
if (rb_obj_is_mutex((VALUE)p)) break;
if (rb_obj_is_fiber((VALUE)p)) break;
p->as.free.flags = 0;
if (RTYPEDDATA_P(p)) {
RDATA(p)->dfree = RANY(p)->as.typeddata.type->function.dfree;
}
if (RANY(p)->as.data.dfree == (RUBY_DATA_FUNC)-1) {
xfree(DATA_PTR(p));
}
else if (RANY(p)->as.data.dfree) {
make_zombie(objspace, (VALUE)p, RANY(p)->as.data.dfree, RANY(p)->as.data.data);
}
break;
case T_FILE:
if (RANY(p)->as.file.fptr) {
make_io_zombie(objspace, (VALUE)p);
}
break;
}
p++;
}
}
gc_exit(objspace, "rb_objspace_call_finalizer");
if (heap_pages_deferred_final) {
finalize_list(objspace, heap_pages_deferred_final);
}
st_free_table(finalizer_table);
finalizer_table = 0;
ATOMIC_SET(finalizing, 0);
}
static inline int
is_id_value(rb_objspace_t *objspace, VALUE ptr)
{
if (!is_pointer_to_heap(objspace, (void *)ptr)) return FALSE;
if (BUILTIN_TYPE(ptr) > T_FIXNUM) return FALSE;
if (BUILTIN_TYPE(ptr) == T_ICLASS) return FALSE;
return TRUE;
}
static inline int
heap_is_swept_object(rb_objspace_t *objspace, rb_heap_t *heap, VALUE ptr)
{
struct heap_page *page = GET_HEAP_PAGE(ptr);
return page->flags.before_sweep ? FALSE : TRUE;
}
static inline int
is_swept_object(rb_objspace_t *objspace, VALUE ptr)
{
if (heap_is_swept_object(objspace, heap_eden, ptr)) {
return TRUE;
}
else {
return FALSE;
}
}
/* garbage objects will be collected soon. */
static inline int
is_garbage_object(rb_objspace_t *objspace, VALUE ptr)
{
if (!is_lazy_sweeping(heap_eden) ||
is_swept_object(objspace, ptr) ||
MARKED_IN_BITMAP(GET_HEAP_MARK_BITS(ptr), ptr)) {
return FALSE;
}
else {
return TRUE;
}
}
static inline int
is_live_object(rb_objspace_t *objspace, VALUE ptr)
{
switch (BUILTIN_TYPE(ptr)) {
case T_NONE:
case T_ZOMBIE:
return FALSE;
}
if (!is_garbage_object(objspace, ptr)) {
return TRUE;
}
else {
return FALSE;
}
}
static inline int
is_markable_object(rb_objspace_t *objspace, VALUE obj)
{
if (rb_special_const_p(obj)) return FALSE; /* special const is not markable */
check_rvalue_consistency(obj);
return TRUE;
}
int
rb_objspace_markable_object_p(VALUE obj)
{
rb_objspace_t *objspace = &rb_objspace;
return is_markable_object(objspace, obj) && is_live_object(objspace, obj);
}
int
rb_objspace_garbage_object_p(VALUE obj)
{
rb_objspace_t *objspace = &rb_objspace;
return is_garbage_object(objspace, obj);
}
/*
* call-seq:
* ObjectSpace._id2ref(object_id) -> an_object
*
* Converts an object id to a reference to the object. May not be
* called on an object id passed as a parameter to a finalizer.
*
* s = "I am a string" #=> "I am a string"
* r = ObjectSpace._id2ref(s.object_id) #=> "I am a string"
* r == s #=> true
*
*/
static VALUE
id2ref(VALUE obj, VALUE objid)
{
#if SIZEOF_LONG == SIZEOF_VOIDP
#define NUM2PTR(x) NUM2ULONG(x)
#elif SIZEOF_LONG_LONG == SIZEOF_VOIDP
#define NUM2PTR(x) NUM2ULL(x)
#endif
rb_objspace_t *objspace = &rb_objspace;
VALUE ptr;
void *p0;
ptr = NUM2PTR(objid);
p0 = (void *)ptr;
if (ptr == Qtrue) return Qtrue;
if (ptr == Qfalse) return Qfalse;
if (ptr == Qnil) return Qnil;
if (FIXNUM_P(ptr)) return (VALUE)ptr;
if (FLONUM_P(ptr)) return (VALUE)ptr;
ptr = obj_id_to_ref(objid);
if ((ptr % sizeof(RVALUE)) == (4 << 2)) {
ID symid = ptr / sizeof(RVALUE);
if (rb_id2str(symid) == 0)
rb_raise(rb_eRangeError, "%p is not symbol id value", p0);
return ID2SYM(symid);
}
if (!is_id_value(objspace, ptr)) {
rb_raise(rb_eRangeError, "%p is not id value", p0);
}
if (!is_live_object(objspace, ptr)) {
rb_raise(rb_eRangeError, "%p is recycled object", p0);
}
if (RBASIC(ptr)->klass == 0) {
rb_raise(rb_eRangeError, "%p is internal object", p0);
}
return (VALUE)ptr;
}
/*
* Document-method: __id__
* Document-method: object_id
*
* call-seq:
* obj.__id__ -> integer
* obj.object_id -> integer
*
* Returns an integer identifier for +obj+.
*
* The same number will be returned on all calls to +object_id+ for a given
* object, and no two active objects will share an id.
*
* Note: that some objects of builtin classes are reused for optimization.
* This is the case for immediate values and frozen string literals.
*
* Immediate values are not passed by reference but are passed by value:
* +nil+, +true+, +false+, Fixnums, Symbols, and some Floats.
*
* Object.new.object_id == Object.new.object_id # => false
* (21 * 2).object_id == (21 * 2).object_id # => true
* "hello".object_id == "hello".object_id # => false
* "hi".freeze.object_id == "hi".freeze.object_id # => true
*/
VALUE
rb_obj_id(VALUE obj)
{
/*
* 32-bit VALUE space
* MSB ------------------------ LSB
* false 00000000000000000000000000000000
* true 00000000000000000000000000000010
* nil 00000000000000000000000000000100
* undef 00000000000000000000000000000110
* symbol ssssssssssssssssssssssss00001110
* object oooooooooooooooooooooooooooooo00 = 0 (mod sizeof(RVALUE))
* fixnum fffffffffffffffffffffffffffffff1
*
* object_id space
* LSB
* false 00000000000000000000000000000000
* true 00000000000000000000000000000010
* nil 00000000000000000000000000000100
* undef 00000000000000000000000000000110
* symbol 000SSSSSSSSSSSSSSSSSSSSSSSSSSS0 S...S % A = 4 (S...S = s...s * A + 4)
* object oooooooooooooooooooooooooooooo0 o...o % A = 0
* fixnum fffffffffffffffffffffffffffffff1 bignum if required
*
* where A = sizeof(RVALUE)/4
*
* sizeof(RVALUE) is
* 20 if 32-bit, double is 4-byte aligned
* 24 if 32-bit, double is 8-byte aligned
* 40 if 64-bit
*/
if (STATIC_SYM_P(obj)) {
return (SYM2ID(obj) * sizeof(RVALUE) + (4 << 2)) | FIXNUM_FLAG;
}
else if (FLONUM_P(obj)) {
#if SIZEOF_LONG == SIZEOF_VOIDP
return LONG2NUM((SIGNED_VALUE)obj);
#else
return LL2NUM((SIGNED_VALUE)obj);
#endif
}
else if (SPECIAL_CONST_P(obj)) {
return LONG2NUM((SIGNED_VALUE)obj);
}
return nonspecial_obj_id(obj);
}
#include "regint.h"
static size_t
obj_memsize_of(VALUE obj, int use_all_types)
{
size_t size = 0;
if (SPECIAL_CONST_P(obj)) {
return 0;
}
if (FL_TEST(obj, FL_EXIVAR)) {
size += rb_generic_ivar_memsize(obj);
}
switch (BUILTIN_TYPE(obj)) {
case T_OBJECT:
if (!(RBASIC(obj)->flags & ROBJECT_EMBED) &&
ROBJECT(obj)->as.heap.ivptr) {
size += ROBJECT(obj)->as.heap.numiv * sizeof(VALUE);
}
break;
case T_MODULE:
case T_CLASS:
if (RCLASS_M_TBL(obj)) {
size += st_memsize(RCLASS_M_TBL(obj));
}
if (RCLASS_EXT(obj)) {
if (RCLASS_IV_TBL(obj)) {
size += st_memsize(RCLASS_IV_TBL(obj));
}
if (RCLASS_IV_INDEX_TBL(obj)) {
size += st_memsize(RCLASS_IV_INDEX_TBL(obj));
}
if (RCLASS(obj)->ptr->iv_tbl) {
size += st_memsize(RCLASS(obj)->ptr->iv_tbl);
}
if (RCLASS(obj)->ptr->const_tbl) {
size += st_memsize(RCLASS(obj)->ptr->const_tbl);
}
size += sizeof(rb_classext_t);
}
break;
case T_ICLASS:
if (FL_TEST(obj, RICLASS_IS_ORIGIN)) {
if (RCLASS_M_TBL(obj)) {
size += st_memsize(RCLASS_M_TBL(obj));
}
}
break;
case T_STRING:
size += rb_str_memsize(obj);
break;
case T_ARRAY:
size += rb_ary_memsize(obj);
break;
case T_HASH:
if (RHASH(obj)->ntbl) {
size += st_memsize(RHASH(obj)->ntbl);
}
break;
case T_REGEXP:
if (RREGEXP(obj)->ptr) {
size += onig_memsize(RREGEXP(obj)->ptr);
}
break;
case T_DATA:
if (use_all_types) size += rb_objspace_data_type_memsize(obj);
break;
case T_MATCH:
if (RMATCH(obj)->rmatch) {
struct rmatch *rm = RMATCH(obj)->rmatch;
size += onig_region_memsize(&rm->regs);
size += sizeof(struct rmatch_offset) * rm->char_offset_num_allocated;
size += sizeof(struct rmatch);
}
break;
case T_FILE:
if (RFILE(obj)->fptr) {
size += rb_io_memsize(RFILE(obj)->fptr);
}
break;
case T_RATIONAL:
case T_COMPLEX:
case T_IMEMO:
break;
case T_FLOAT:
case T_SYMBOL:
break;
case T_BIGNUM:
if (!(RBASIC(obj)->flags & BIGNUM_EMBED_FLAG) && BIGNUM_DIGITS(obj)) {
size += BIGNUM_LEN(obj) * sizeof(BDIGIT);
}
break;
case T_NODE:
if (use_all_types) size += rb_node_memsize(obj);
break;
case T_STRUCT:
if ((RBASIC(obj)->flags & RSTRUCT_EMBED_LEN_MASK) == 0 &&
RSTRUCT(obj)->as.heap.ptr) {
size += sizeof(VALUE) * RSTRUCT_LEN(obj);
}
break;
case T_ZOMBIE:
break;
default:
rb_bug("objspace/memsize_of(): unknown data type 0x%x(%p)",
BUILTIN_TYPE(obj), (void*)obj);
}
return size + sizeof(RVALUE);
}
size_t
rb_obj_memsize_of(VALUE obj)
{
return obj_memsize_of(obj, TRUE);
}
static int
set_zero(st_data_t key, st_data_t val, st_data_t arg)
{
VALUE k = (VALUE)key;
VALUE hash = (VALUE)arg;
rb_hash_aset(hash, k, INT2FIX(0));
return ST_CONTINUE;
}
/*
* call-seq:
* ObjectSpace.count_objects([result_hash]) -> hash
*
* Counts objects for each type.
*
* It returns a hash, such as:
* {
* :TOTAL=>10000,
* :FREE=>3011,
* :T_OBJECT=>6,
* :T_CLASS=>404,
* # ...
* }
*
* The contents of the returned hash are implementation specific.
* It may be changed in future.
*
* If the optional argument +result_hash+ is given,
* it is overwritten and returned. This is intended to avoid probe effect.
*
* This method is only expected to work on C Ruby.
*
*/
static VALUE
count_objects(int argc, VALUE *argv, VALUE os)
{
rb_objspace_t *objspace = &rb_objspace;
size_t counts[T_MASK+1];
size_t freed = 0;
size_t total = 0;
size_t i;
VALUE hash;
if (rb_scan_args(argc, argv, "01", &hash) == 1) {
if (!RB_TYPE_P(hash, T_HASH))
rb_raise(rb_eTypeError, "non-hash given");
}
for (i = 0; i <= T_MASK; i++) {
counts[i] = 0;
}
for (i = 0; i < heap_allocated_pages; i++) {
struct heap_page *page = heap_pages_sorted[i];
RVALUE *p, *pend;
p = page->start; pend = p + page->total_slots;
for (;p < pend; p++) {
if (p->as.basic.flags) {
counts[BUILTIN_TYPE(p)]++;
}
else {
freed++;
}
}
total += page->total_slots;
}
if (hash == Qnil) {
hash = rb_hash_new();
}
else if (!RHASH_EMPTY_P(hash)) {
st_foreach(RHASH_TBL_RAW(hash), set_zero, hash);
}
rb_hash_aset(hash, ID2SYM(rb_intern("TOTAL")), SIZET2NUM(total));
rb_hash_aset(hash, ID2SYM(rb_intern("FREE")), SIZET2NUM(freed));
for (i = 0; i <= T_MASK; i++) {
VALUE type;
switch (i) {
#define COUNT_TYPE(t) case (t): type = ID2SYM(rb_intern(#t)); break;
COUNT_TYPE(T_NONE);
COUNT_TYPE(T_OBJECT);
COUNT_TYPE(T_CLASS);
COUNT_TYPE(T_MODULE);
COUNT_TYPE(T_FLOAT);
COUNT_TYPE(T_STRING);
COUNT_TYPE(T_REGEXP);
COUNT_TYPE(T_ARRAY);
COUNT_TYPE(T_HASH);
COUNT_TYPE(T_STRUCT);
COUNT_TYPE(T_BIGNUM);
COUNT_TYPE(T_FILE);
COUNT_TYPE(T_DATA);
COUNT_TYPE(T_MATCH);
COUNT_TYPE(T_COMPLEX);
COUNT_TYPE(T_RATIONAL);
COUNT_TYPE(T_NIL);
COUNT_TYPE(T_TRUE);
COUNT_TYPE(T_FALSE);
COUNT_TYPE(T_SYMBOL);
COUNT_TYPE(T_FIXNUM);
COUNT_TYPE(T_IMEMO);
COUNT_TYPE(T_UNDEF);
COUNT_TYPE(T_NODE);
COUNT_TYPE(T_ICLASS);
COUNT_TYPE(T_ZOMBIE);
#undef COUNT_TYPE
default: type = INT2NUM(i); break;
}
if (counts[i])
rb_hash_aset(hash, type, SIZET2NUM(counts[i]));
}
return hash;
}
/*
------------------------ Garbage Collection ------------------------
*/
/* Sweeping */
static size_t
objspace_available_slots(rb_objspace_t *objspace)
{
return heap_eden->total_slots + heap_tomb->total_slots;
}
static size_t
objspace_live_slots(rb_objspace_t *objspace)
{
return (objspace->total_allocated_objects - objspace->profile.total_freed_objects) - heap_pages_final_slots;
}
static size_t
objspace_free_slots(rb_objspace_t *objspace)
{
return objspace_available_slots(objspace) - objspace_live_slots(objspace) - heap_pages_final_slots;
}
static void
gc_setup_mark_bits(struct heap_page *page)
{
#if USE_RGENGC
/* copy oldgen bitmap to mark bitmap */
memcpy(&page->mark_bits[0], &page->uncollectible_bits[0], HEAP_BITMAP_SIZE);
#else
/* clear mark bitmap */
memset(&page->mark_bits[0], 0, HEAP_BITMAP_SIZE);
#endif
}
/* TRUE : has empty slots */
/* FALSE: no empty slots (or move to tomb heap because no live slots) */
static inline void
gc_page_sweep(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *sweep_page)
{
int i;
int empty_slots = 0, freed_slots = 0, final_slots = 0;
RVALUE *p, *pend,*offset;
bits_t *bits, bitset;
gc_report(2, objspace, "page_sweep: start.\n");
sweep_page->flags.before_sweep = FALSE;
p = sweep_page->start; pend = p + sweep_page->total_slots;
offset = p - NUM_IN_PAGE(p);
bits = sweep_page->mark_bits;
/* create guard : fill 1 out-of-range */
bits[BITMAP_INDEX(p)] |= BITMAP_BIT(p)-1;
bits[BITMAP_INDEX(pend)] |= ~(BITMAP_BIT(pend) - 1);
for (i=0; i < HEAP_BITMAP_LIMIT; i++) {
bitset = ~bits[i];
if (bitset) {
p = offset + i * BITS_BITLENGTH;
do {
if (bitset & 1) {
switch (BUILTIN_TYPE(p)) {
default: { /* majority case */
gc_report(2, objspace, "page_sweep: free %s\n", obj_info((VALUE)p));
#if USE_RGENGC && RGENGC_CHECK_MODE
if (!is_full_marking(objspace)) {
if (RVALUE_OLD_P((VALUE)p)) rb_bug("page_sweep: %s - old while minor GC.", obj_info((VALUE)p));
if (rgengc_remembered(objspace, (VALUE)p)) rb_bug("page_sweep: %s - remembered.", obj_info((VALUE)p));
}
#endif
if (obj_free(objspace, (VALUE)p)) {
final_slots++;
}
else {
(void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, sizeof(RVALUE));
heap_page_add_freeobj(objspace, sweep_page, (VALUE)p);
gc_report(3, objspace, "page_sweep: %s is added to freelist\n", obj_info((VALUE)p));
freed_slots++;
}
break;
}
/* minor cases */
case T_ZOMBIE:
/* already counted */
break;
case T_NONE:
empty_slots++; /* already freed */
break;
}
}
p++;
bitset >>= 1;
} while (bitset);
}
}
gc_setup_mark_bits(sweep_page);
#if GC_PROFILE_MORE_DETAIL
if (gc_prof_enabled(objspace)) {
gc_profile_record *record = gc_prof_record(objspace);
record->removing_objects += final_slots + freed_slots;
record->empty_objects += empty_slots;
}
#endif
if (0) fprintf(stderr, "gc_page_sweep(%d): total_slots: %d, freed_slots: %d, empty_slots: %d, final_slots: %d\n",
(int)rb_gc_count(),
(int)sweep_page->total_slots,
freed_slots, empty_slots, final_slots);
heap_pages_swept_slots += sweep_page->free_slots = freed_slots + empty_slots;
objspace->profile.total_freed_objects += freed_slots;
heap_pages_final_slots += final_slots;
sweep_page->final_slots += final_slots;
if (heap_pages_deferred_final && !finalizing) {
rb_thread_t *th = GET_THREAD();
if (th) {
gc_finalize_deferred_register();
}
}
gc_report(2, objspace, "page_sweep: end.\n");
}
/* allocate additional minimum page to work */
static void
gc_heap_prepare_minimum_pages(rb_objspace_t *objspace, rb_heap_t *heap)
{
if (!heap->free_pages && heap_increment(objspace, heap) == FALSE) {
/* there is no free after page_sweep() */
heap_set_increment(objspace, 1);
if (!heap_increment(objspace, heap)) { /* can't allocate additional free objects */
rb_memerror();
}
}
}
static void
gc_stat_transition(rb_objspace_t *objspace, enum gc_stat stat)
{
#if RGENGC_CHECK_MODE
enum gc_stat prev_stat = objspace->flags.stat;
switch (prev_stat) {
case gc_stat_none: assert(stat == gc_stat_marking); break;
case gc_stat_marking: assert(stat == gc_stat_sweeping); break;
case gc_stat_sweeping: assert(stat == gc_stat_none); break;
}
#endif
objspace->flags.stat = stat;
}
static void
gc_sweep_start_heap(rb_objspace_t *objspace, rb_heap_t *heap)
{
heap->sweep_pages = heap->pages;
heap->free_pages = NULL;
#if GC_ENABLE_INCREMENTAL_MARK
heap->pooled_pages = NULL;
objspace->rincgc.pooled_slots = 0;
#endif
if (heap->using_page) {
RVALUE **p = &heap->using_page->freelist;
while (*p) {
p = &(*p)->as.free.next;
}
*p = heap->freelist;
heap->using_page = NULL;
}
heap->freelist = NULL;
}
#if defined(__GNUC__) && __GNUC__ == 4 && __GNUC_MINOR__ == 4
__attribute__((noinline))
#endif
static void
gc_sweep_start(rb_objspace_t *objspace)
{
rb_heap_t *heap;
size_t total_limit_slot;
gc_stat_transition(objspace, gc_stat_sweeping);
/* sweep unlinked method entries */
if (GET_VM()->unlinked_method_entry_list) {
rb_sweep_method_entry(GET_VM());
}
/* sometimes heap_allocatable_pages is not 0 */
heap_pages_swept_slots = heap_allocatable_pages * HEAP_OBJ_LIMIT;
total_limit_slot = objspace_available_slots(objspace);
heap_pages_min_free_slots = (size_t)(total_limit_slot * GC_HEAP_FREE_SLOTS_MIN_RATIO);
if (heap_pages_min_free_slots < gc_params.heap_free_slots) {
heap_pages_min_free_slots = gc_params.heap_free_slots;
}
heap_pages_max_free_slots = (size_t)(total_limit_slot * GC_HEAP_FREE_SLOTS_MAX_RATIO);
if (heap_pages_max_free_slots < gc_params.heap_init_slots) {
heap_pages_max_free_slots = gc_params.heap_init_slots;
}
if (0) fprintf(stderr, "heap_pages_min_free_slots: %d, heap_pages_max_free_slots: %d\n",
(int)heap_pages_min_free_slots, (int)heap_pages_max_free_slots);
heap = heap_eden;
gc_sweep_start_heap(objspace, heap);
}
static void
gc_sweep_finish(rb_objspace_t *objspace)
{
rb_heap_t *heap = heap_eden;
gc_report(1, objspace, "gc_sweep_finish: heap->total_slots: %d, heap->swept_slots: %d, min_free_slots: %d\n",
(int)heap->total_slots, (int)heap_pages_swept_slots, (int)heap_pages_min_free_slots);
gc_prof_set_heap_info(objspace);
heap_pages_free_unused_pages(objspace);
/* if heap_pages has unused pages, then assign them to increment */
if (heap_allocatable_pages < heap_tomb->page_length) {
heap_allocatable_pages = heap_tomb->page_length;
}
gc_event_hook(objspace, RUBY_INTERNAL_EVENT_GC_END_SWEEP, 0);
gc_stat_transition(objspace, gc_stat_none);
#if RGENGC_CHECK_MODE >= 2
gc_verify_internal_consistency(Qnil);
#endif
}
static int
gc_sweep_step(rb_objspace_t *objspace, rb_heap_t *heap)
{
struct heap_page *sweep_page = heap->sweep_pages, *next;
int unlink_limit = 3;
#if GC_ENABLE_INCREMENTAL_MARK
int need_pool = will_be_incremental_marking(objspace) ? TRUE : FALSE;
gc_report(2, objspace, "gc_sweep_step (need_pool: %d)\n", need_pool);
#else
gc_report(2, objspace, "gc_sweep_step\n");
#endif
if (sweep_page == NULL) return FALSE;
#if GC_ENABLE_LAZY_SWEEP
gc_prof_sweep_timer_start(objspace);
#endif
while (sweep_page) {
heap->sweep_pages = next = sweep_page->next;
gc_page_sweep(objspace, heap, sweep_page);
if (sweep_page->final_slots + sweep_page->free_slots == sweep_page->total_slots &&
unlink_limit > 0) {
unlink_limit--;
/* there are no living objects -> move this page to tomb heap */
heap_unlink_page(objspace, heap, sweep_page);
heap_add_page(objspace, heap_tomb, sweep_page);
}
else if (sweep_page->free_slots > 0) {
#if GC_ENABLE_INCREMENTAL_MARK
if (need_pool) {
if (heap_add_poolpage(objspace, heap, sweep_page)) {
need_pool = FALSE;
}
}
else {
heap_add_freepage(objspace, heap, sweep_page);
break;
}
#else
heap_add_freepage(objspace, heap, sweep_page);
break;
#endif
}
else {
sweep_page->free_next = NULL;
}
sweep_page = next;
}
if (heap->sweep_pages == NULL) {
gc_sweep_finish(objspace);
}
#if GC_ENABLE_LAZY_SWEEP
gc_prof_sweep_timer_stop(objspace);
#endif
return heap->free_pages != NULL;
}
static void
gc_sweep_rest(rb_objspace_t *objspace)
{
rb_heap_t *heap = heap_eden; /* lazy sweep only for eden */
while (has_sweeping_pages(heap)) {
gc_sweep_step(objspace, heap);
}
}
#if GC_ENABLE_LAZY_SWEEP
static void
gc_sweep_continue(rb_objspace_t *objspace, rb_heap_t *heap)
{
if (RGENGC_CHECK_MODE) assert(dont_gc == FALSE);
gc_enter(objspace, "sweep_continue");
#if USE_RGENGC
if (objspace->rgengc.need_major_gc == GPR_FLAG_NONE && heap_increment(objspace, heap)) {
gc_report(3, objspace, "gc_sweep_continue: success heap_increment().\n");
}
#endif
gc_sweep_step(objspace, heap);
gc_exit(objspace, "sweep_continue");
}
#endif
static void
gc_sweep(rb_objspace_t *objspace)
{
const unsigned int immediate_sweep = objspace->flags.immediate_sweep;
gc_report(1, objspace, "gc_sweep: immediate: %d\n", immediate_sweep);
if (immediate_sweep) {
#if !GC_ENABLE_LAZY_SWEEP
gc_prof_sweep_timer_start(objspace);
#endif
gc_sweep_start(objspace);
gc_sweep_rest(objspace);
#if !GC_ENABLE_LAZY_SWEEP
gc_prof_sweep_timer_stop(objspace);
#endif
}
else {
struct heap_page *page;
gc_sweep_start(objspace);
page = heap_eden->sweep_pages;
while (page) {
page->flags.before_sweep = TRUE;
page = page->next;
}
gc_sweep_step(objspace, heap_eden);
}
gc_heap_prepare_minimum_pages(objspace, heap_eden);
}
/* Marking - Marking stack */
static stack_chunk_t *
stack_chunk_alloc(void)
{
stack_chunk_t *res;
res = malloc(sizeof(stack_chunk_t));
if (!res)
rb_memerror();
return res;
}
static inline int
is_mark_stack_empty(mark_stack_t *stack)
{
return stack->chunk == NULL;
}
static size_t
mark_stack_size(mark_stack_t *stack)
{
size_t size = stack->index;
stack_chunk_t *chunk = stack->chunk ? stack->chunk->next : NULL;
while (chunk) {
size += stack->limit;
chunk = chunk->next;
}
return size;
}
static void
add_stack_chunk_cache(mark_stack_t *stack, stack_chunk_t *chunk)
{
chunk->next = stack->cache;
stack->cache = chunk;
stack->cache_size++;
}
static void
shrink_stack_chunk_cache(mark_stack_t *stack)
{
stack_chunk_t *chunk;
if (stack->unused_cache_size > (stack->cache_size/2)) {
chunk = stack->cache;
stack->cache = stack->cache->next;
stack->cache_size--;
free(chunk);
}
stack->unused_cache_size = stack->cache_size;
}
static void
push_mark_stack_chunk(mark_stack_t *stack)
{
stack_chunk_t *next;
if (RGENGC_CHECK_MODE) assert(stack->index == stack->limit);
if (stack->cache_size > 0) {
next = stack->cache;
stack->cache = stack->cache->next;
stack->cache_size--;
if (stack->unused_cache_size > stack->cache_size)
stack->unused_cache_size = stack->cache_size;
}
else {
next = stack_chunk_alloc();
}
next->next = stack->chunk;
stack->chunk = next;
stack->index = 0;
}
static void
pop_mark_stack_chunk(mark_stack_t *stack)
{
stack_chunk_t *prev;
prev = stack->chunk->next;
if (RGENGC_CHECK_MODE) assert(stack->index == 0);
add_stack_chunk_cache(stack, stack->chunk);
stack->chunk = prev;
stack->index = stack->limit;
}
#if (defined(ENABLE_VM_OBJSPACE) && ENABLE_VM_OBJSPACE) || (RGENGC_CHECK_MODE >= 4)
static void
free_stack_chunks(mark_stack_t *stack)
{
stack_chunk_t *chunk = stack->chunk;
stack_chunk_t *next = NULL;
while (chunk != NULL) {
next = chunk->next;
free(chunk);
chunk = next;
}
}
#endif
static void
push_mark_stack(mark_stack_t *stack, VALUE data)
{
if (stack->index == stack->limit) {
push_mark_stack_chunk(stack);
}
stack->chunk->data[stack->index++] = data;
}
static int
pop_mark_stack(mark_stack_t *stack, VALUE *data)
{
if (is_mark_stack_empty(stack)) {
return FALSE;
}
if (stack->index == 1) {
*data = stack->chunk->data[--stack->index];
pop_mark_stack_chunk(stack);
}
else {
*data = stack->chunk->data[--stack->index];
}
return TRUE;
}
#if GC_ENABLE_INCREMENTAL_MARK
static int
invalidate_mark_stack_chunk(stack_chunk_t *chunk, int limit, VALUE obj)
{
int i;
for (i=0; i<limit; i++) {
if (chunk->data[i] == obj) {
chunk->data[i] = Qundef;
return TRUE;
}
}
return FALSE;
}
static void
invalidate_mark_stack(mark_stack_t *stack, VALUE obj)
{
stack_chunk_t *chunk = stack->chunk;
int limit = stack->index;
while (chunk) {
if (invalidate_mark_stack_chunk(chunk, limit, obj)) return;
chunk = chunk->next;
limit = stack->limit;
}
rb_bug("invalid_mark_stack: unreachable");
}
#endif
static void
init_mark_stack(mark_stack_t *stack)
{
int i;
MEMZERO(stack, mark_stack_t, 1);
stack->index = stack->limit = STACK_CHUNK_SIZE;
stack->cache_size = 0;
for (i=0; i < 4; i++) {
add_stack_chunk_cache(stack, stack_chunk_alloc());
}
stack->unused_cache_size = stack->cache_size;
}
/* Marking */
#ifdef __ia64
#define SET_STACK_END (SET_MACHINE_STACK_END(&th->machine.stack_end), th->machine.register_stack_end = rb_ia64_bsp())
#else
#define SET_STACK_END SET_MACHINE_STACK_END(&th->machine.stack_end)
#endif
#define STACK_START (th->machine.stack_start)
#define STACK_END (th->machine.stack_end)
#define STACK_LEVEL_MAX (th->machine.stack_maxsize/sizeof(VALUE))
#if STACK_GROW_DIRECTION < 0
# define STACK_LENGTH (size_t)(STACK_START - STACK_END)
#elif STACK_GROW_DIRECTION > 0
# define STACK_LENGTH (size_t)(STACK_END - STACK_START + 1)
#else
# define STACK_LENGTH ((STACK_END < STACK_START) ? (size_t)(STACK_START - STACK_END) \
: (size_t)(STACK_END - STACK_START + 1))
#endif
#if !STACK_GROW_DIRECTION
int ruby_stack_grow_direction;
int
ruby_get_stack_grow_direction(volatile VALUE *addr)
{
VALUE *end;
SET_MACHINE_STACK_END(&end);
if (end > addr) return ruby_stack_grow_direction = 1;
return ruby_stack_grow_direction = -1;
}
#endif
size_t
ruby_stack_length(VALUE **p)
{
rb_thread_t *th = GET_THREAD();
SET_STACK_END;
if (p) *p = STACK_UPPER(STACK_END, STACK_START, STACK_END);
return STACK_LENGTH;
}
#if !(defined(POSIX_SIGNAL) && defined(SIGSEGV) && defined(HAVE_SIGALTSTACK))
static int
stack_check(int water_mark)
{
int ret;
rb_thread_t *th = GET_THREAD();
SET_STACK_END;
ret = STACK_LENGTH > STACK_LEVEL_MAX - water_mark;
#ifdef __ia64
if (!ret) {
ret = (VALUE*)rb_ia64_bsp() - th->machine.register_stack_start >
th->machine.register_stack_maxsize/sizeof(VALUE) - water_mark;
}
#endif
return ret;
}
#endif
#define STACKFRAME_FOR_CALL_CFUNC 512
int
ruby_stack_check(void)
{
#if defined(POSIX_SIGNAL) && defined(SIGSEGV) && defined(HAVE_SIGALTSTACK)
return 0;
#else
return stack_check(STACKFRAME_FOR_CALL_CFUNC);
#endif
}
ATTRIBUTE_NO_ADDRESS_SAFETY_ANALYSIS
static void
mark_locations_array(rb_objspace_t *objspace, register const VALUE *x, register long n)
{
VALUE v;
while (n--) {
v = *x;
gc_mark_maybe(objspace, v);
x++;
}
}
static void
gc_mark_locations(rb_objspace_t *objspace, const VALUE *start, const VALUE *end)
{
long n;
if (end <= start) return;
n = end - start;
mark_locations_array(objspace, start, n);
}
void
rb_gc_mark_locations(const VALUE *start, const VALUE *end)
{
gc_mark_locations(&rb_objspace, start, end);
}
void
rb_gc_mark_values(long n, const VALUE *values)
{
rb_objspace_t *objspace = &rb_objspace;
long i;
for (i=0; i<n; i++) {
gc_mark(objspace, values[i]);
}
}
#define rb_gc_mark_locations(start, end) gc_mark_locations(objspace, (start), (end))
struct mark_tbl_arg {
rb_objspace_t *objspace;
};
static int
mark_entry(st_data_t key, st_data_t value, st_data_t data)
{
struct mark_tbl_arg *arg = (void*)data;
gc_mark(arg->objspace, (VALUE)value);
return ST_CONTINUE;
}
static void
mark_tbl(rb_objspace_t *objspace, st_table *tbl)
{
struct mark_tbl_arg arg;
if (!tbl || tbl->num_entries == 0) return;
arg.objspace = objspace;
st_foreach(tbl, mark_entry, (st_data_t)&arg);
}
static int
mark_key(st_data_t key, st_data_t value, st_data_t data)
{
struct mark_tbl_arg *arg = (void*)data;
gc_mark(arg->objspace, (VALUE)key);
return ST_CONTINUE;
}
static void
mark_set(rb_objspace_t *objspace, st_table *tbl)
{
struct mark_tbl_arg arg;
if (!tbl) return;
arg.objspace = objspace;
st_foreach(tbl, mark_key, (st_data_t)&arg);
}
void
rb_mark_set(st_table *tbl)
{
mark_set(&rb_objspace, tbl);
}
static int
mark_keyvalue(st_data_t key, st_data_t value, st_data_t data)
{
struct mark_tbl_arg *arg = (void*)data;
gc_mark(arg->objspace, (VALUE)key);
gc_mark(arg->objspace, (VALUE)value);
return ST_CONTINUE;
}
static void
mark_hash(rb_objspace_t *objspace, st_table *tbl)
{
struct mark_tbl_arg arg;
if (!tbl) return;
arg.objspace = objspace;
st_foreach(tbl, mark_keyvalue, (st_data_t)&arg);
}
void
rb_mark_hash(st_table *tbl)
{
mark_hash(&rb_objspace, tbl);
}
static void
mark_method_entry(rb_objspace_t *objspace, const rb_method_entry_t *me)
{
const rb_method_definition_t *def = me->def;
gc_mark(objspace, me->klass);
again:
if (!def) return;
switch (def->type) {
case VM_METHOD_TYPE_ISEQ:
gc_mark(objspace, def->body.iseq_body.iseq->self);
gc_mark(objspace, (VALUE)def->body.iseq_body.cref);
break;
case VM_METHOD_TYPE_BMETHOD:
gc_mark(objspace, def->body.proc);
break;
case VM_METHOD_TYPE_ATTRSET:
case VM_METHOD_TYPE_IVAR:
gc_mark(objspace, def->body.attr.location);
break;
case VM_METHOD_TYPE_REFINED:
if (def->body.orig_me) {
def = def->body.orig_me->def;
goto again;
}
break;
default:
break; /* ignore */
}
}
void
rb_mark_method_entry(const rb_method_entry_t *me)
{
mark_method_entry(&rb_objspace, me);
}
static int
mark_method_entry_i(st_data_t key, st_data_t value, st_data_t data)
{
const rb_method_entry_t *me = (const rb_method_entry_t *)value;
struct mark_tbl_arg *arg = (void*)data;
mark_method_entry(arg->objspace, me);
return ST_CONTINUE;
}
static void
mark_m_tbl(rb_objspace_t *objspace, struct st_table *tbl)
{
if (tbl) {
struct mark_tbl_arg arg;
arg.objspace = objspace;
st_foreach(tbl, mark_method_entry_i, (st_data_t)&arg);
}
}
static int
mark_const_entry_i(st_data_t key, st_data_t value, st_data_t data)
{
const rb_const_entry_t *ce = (const rb_const_entry_t *)value;
struct mark_tbl_arg *arg = (void*)data;
gc_mark(arg->objspace, ce->value);
gc_mark(arg->objspace, ce->file);
return ST_CONTINUE;
}
static void
mark_const_tbl(rb_objspace_t *objspace, st_table *tbl)
{
struct mark_tbl_arg arg;
if (!tbl) return;
arg.objspace = objspace;
st_foreach(tbl, mark_const_entry_i, (st_data_t)&arg);
}
#if STACK_GROW_DIRECTION < 0
#define GET_STACK_BOUNDS(start, end, appendix) ((start) = STACK_END, (end) = STACK_START)
#elif STACK_GROW_DIRECTION > 0
#define GET_STACK_BOUNDS(start, end, appendix) ((start) = STACK_START, (end) = STACK_END+(appendix))
#else
#define GET_STACK_BOUNDS(start, end, appendix) \
((STACK_END < STACK_START) ? \
((start) = STACK_END, (end) = STACK_START) : ((start) = STACK_START, (end) = STACK_END+(appendix)))
#endif
static void
mark_current_machine_context(rb_objspace_t *objspace, rb_thread_t *th)
{
union {
rb_jmp_buf j;
VALUE v[sizeof(rb_jmp_buf) / sizeof(VALUE)];
} save_regs_gc_mark;
VALUE *stack_start, *stack_end;
FLUSH_REGISTER_WINDOWS;
/* This assumes that all registers are saved into the jmp_buf (and stack) */
rb_setjmp(save_regs_gc_mark.j);
/* SET_STACK_END must be called in this function because
* the stack frame of this function may contain
* callee save registers and they should be marked. */
SET_STACK_END;
GET_STACK_BOUNDS(stack_start, stack_end, 1);
mark_locations_array(objspace, save_regs_gc_mark.v, numberof(save_regs_gc_mark.v));
rb_gc_mark_locations(stack_start, stack_end);
#ifdef __ia64
rb_gc_mark_locations(th->machine.register_stack_start, th->machine.register_stack_end);
#endif
#if defined(__mc68000__)
rb_gc_mark_locations((VALUE*)((char*)stack_start + 2),
(VALUE*)((char*)stack_end - 2));
#endif
}
void
rb_gc_mark_machine_stack(rb_thread_t *th)
{
rb_objspace_t *objspace = &rb_objspace;
VALUE *stack_start, *stack_end;
GET_STACK_BOUNDS(stack_start, stack_end, 0);
rb_gc_mark_locations(stack_start, stack_end);
#ifdef __ia64
rb_gc_mark_locations(th->machine.register_stack_start, th->machine.register_stack_end);
#endif
#if defined(__mc68000__)
rb_gc_mark_locations((VALUE*)((char*)stack_start + 2),
(VALUE*)((char*)stack_end - 2));
#endif
}
void
rb_mark_tbl(st_table *tbl)
{
mark_tbl(&rb_objspace, tbl);
}
static void
gc_mark_maybe(rb_objspace_t *objspace, VALUE obj)
{
(void)VALGRIND_MAKE_MEM_DEFINED(&obj, sizeof(obj));
if (is_pointer_to_heap(objspace, (void *)obj)) {
int type = BUILTIN_TYPE(obj);
if (type != T_ZOMBIE && type != T_NONE) {
gc_mark_ptr(objspace, obj);
}
}
}
void
rb_gc_mark_maybe(VALUE obj)
{
gc_mark_maybe(&rb_objspace, obj);
}
static inline int
gc_mark_set(rb_objspace_t *objspace, VALUE obj)
{
if (RVALUE_MARKED(obj)) return 0;
MARK_IN_BITMAP(GET_HEAP_MARK_BITS(obj), obj);
return 1;
}
#if USE_RGENGC
static int
gc_remember_unprotected(rb_objspace_t *objspace, VALUE obj)
{
struct heap_page *page = GET_HEAP_PAGE(obj);
bits_t *uncollectible_bits = &page->uncollectible_bits[0];
if (!MARKED_IN_BITMAP(uncollectible_bits, obj)) {
page->flags.has_uncollectible_shady_objects = TRUE;
MARK_IN_BITMAP(uncollectible_bits, obj);
objspace->rgengc.uncollectible_wb_unprotected_objects++;
#if RGENGC_PROFILE > 0
objspace->profile.total_remembered_shady_object_count++;
#if RGENGC_PROFILE >= 2
objspace->profile.remembered_shady_object_count_types[BUILTIN_TYPE(obj)]++;
#endif
#endif
return TRUE;
}
else {
return FALSE;
}
}
#endif
static void
rgengc_check_relation(rb_objspace_t *objspace, VALUE obj)
{
#if USE_RGENGC
const VALUE old_parent = objspace->rgengc.parent_object;
if (old_parent) { /* parent object is old */
if (RVALUE_WB_UNPROTECTED(obj)) {
if (gc_remember_unprotected(objspace, obj)) {
gc_report(2, objspace, "relation: (O->S) %s -> %s\n", obj_info(old_parent), obj_info(obj));
}
}
else {
if (!RVALUE_OLD_P(obj)) {
if (RVALUE_MARKED(obj)) {
/* An object pointed from an OLD object should be OLD. */
gc_report(2, objspace, "relation: (O->unmarked Y) %s -> %s\n", obj_info(old_parent), obj_info(obj));
RVALUE_AGE_SET_OLD(objspace, obj);
if (is_incremental_marking(objspace)) {
if (!RVALUE_MARKING(obj)) {
gc_grey(objspace, obj);
}
}
else {
rgengc_remember(objspace, obj);
}
}
else {
gc_report(2, objspace, "relation: (O->Y) %s -> %s\n", obj_info(old_parent), obj_info(obj));
RVALUE_AGE_SET_CANDIDATE(objspace, obj);
}
}
}
}
if (RGENGC_CHECK_MODE) assert(old_parent == objspace->rgengc.parent_object);
#endif
}
static void
gc_grey(rb_objspace_t *objspace, VALUE obj)
{
#if RGENGC_CHECK_MODE
if (RVALUE_MARKED(obj) == FALSE) rb_bug("gc_grey: %s is not marked.", obj_info(obj));
if (RVALUE_MARKING(obj) == TRUE) rb_bug("gc_grey: %s is marking/remembered.", obj_info(obj));
#endif
#if GC_ENABLE_INCREMENTAL_MARK
if (is_incremental_marking(objspace)) {
MARK_IN_BITMAP(GET_HEAP_MARKING_BITS(obj), obj);
}
#endif
push_mark_stack(&objspace->mark_stack, obj);
}
static void
gc_aging(rb_objspace_t *objspace, VALUE obj)
{
#if USE_RGENGC
struct heap_page *page = GET_HEAP_PAGE(obj);
#if RGENGC_CHECK_MODE
assert(RVALUE_MARKING(obj) == FALSE);
#endif
check_rvalue_consistency(obj);
if (!RVALUE_PAGE_WB_UNPROTECTED(page, obj)) {
if (!RVALUE_OLD_P(obj)) {
gc_report(3, objspace, "gc_aging: YOUNG: %s\n", obj_info(obj));
RVALUE_AGE_INC(objspace, obj);
}
else if (is_full_marking(objspace)) {
if (RGENGC_CHECK_MODE) assert(RVALUE_PAGE_UNCOLLECTIBLE(page, obj) == FALSE);
RVALUE_PAGE_OLD_UNCOLLECTIBLE_SET(objspace, page, obj);
}
}
check_rvalue_consistency(obj);
#endif /* USE_RGENGC */
objspace->marked_slots++;
}
static void
gc_mark_ptr(rb_objspace_t *objspace, VALUE obj)
{
if (LIKELY(objspace->mark_func_data == NULL)) {
rgengc_check_relation(objspace, obj);
if (!gc_mark_set(objspace, obj)) return; /* already marked */
gc_aging(objspace, obj);
gc_grey(objspace, obj);
}
else {
objspace->mark_func_data->mark_func(obj, objspace->mark_func_data->data);
}
}
static void
gc_mark(rb_objspace_t *objspace, VALUE obj)
{
if (!is_markable_object(objspace, obj)) return;
gc_mark_ptr(objspace, obj);
}
void
rb_gc_mark(VALUE ptr)
{
gc_mark(&rb_objspace, ptr);
}
/* CAUTION: THIS FUNCTION ENABLE *ONLY BEFORE* SWEEPING.
* This function is only for GC_END_MARK timing.
*/
int
rb_objspace_marked_object_p(VALUE obj)
{
return RVALUE_MARKED(obj) ? TRUE : FALSE;
}