Fix crash in tracing object allocations

ObjectSpace.trace_object_allocations_start could crash since it adds a
TracePoint for when objects are freed. However, TracePoint could crash
since it modifies st tables while inside the GC that is trying to free
the object. This could cause a memory allocation to happen which would
crash if it triggers another GC.

See a crash log: http://ci.rvm.jp/results/trunk@ruby-sp1/4373707

Author: peterzhu2118

ext/objspace/depend

@@ -158,6 +158,7 @@ object_tracing.o: $(hdrdir)/ruby/missing.h
 object_tracing.o: $(hdrdir)/ruby/ruby.h
 object_tracing.o: $(hdrdir)/ruby/st.h
 object_tracing.o: $(hdrdir)/ruby/subst.h
+object_tracing.o: $(top_srcdir)/gc.h
 object_tracing.o: $(top_srcdir)/internal.h
 object_tracing.o: object_tracing.c
 object_tracing.o: objspace.h

ext/objspace/object_tracing.c

@@ -13,6 +13,7 @@
 
 **********************************************************************/
 
+#include "gc.h"
 #include "internal.h"
 #include "ruby/debug.h"
 #include "objspace.h"
@@ -121,6 +122,10 @@ freeobj_i(VALUE tpval, void *data)
     st_data_t v;
     struct allocation_info *info;
 
+    /* Modifying the st table can cause allocations, which can trigger GC.
+     * Since freeobj_i is called during GC, it must not trigger another GC. */
+    VALUE gc_disabled = rb_gc_disable_no_rest();
+
     if (arg->keep_remains) {
         if (st_lookup(arg->object_table, obj, &v)) {
             info = (struct allocation_info *)v;
@@ -135,6 +140,8 @@ freeobj_i(VALUE tpval, void *data)
             ruby_xfree(info);
         }
     }
+
+    if (gc_disabled == Qfalse) rb_gc_enable();
 }
 
 static int

gc.h

@@ -116,8 +116,6 @@ int ruby_get_stack_grow_direction(volatile VALUE *addr);
 const char *rb_obj_info(VALUE obj);
 const char *rb_raw_obj_info(char *const buff, const size_t buff_size, VALUE obj);
 
-VALUE rb_gc_disable_no_rest(void);
-
 struct rb_thread_struct;
 
 size_t rb_size_pool_slot_size(unsigned char pool_id);
@@ -142,6 +140,8 @@ void rb_objspace_each_objects_without_setup(
 
 size_t rb_gc_obj_slot_size(VALUE obj);
 
+VALUE rb_gc_disable_no_rest(void);
+
 RUBY_SYMBOL_EXPORT_END
 
 #endif /* RUBY_GC_H */