Refactor vm->mark_object_ary to be a linked list of WB protected objects

bignum.c

@@ -4728,7 +4728,7 @@ power_cache_get_power(int base, int power_level, size_t *numdigits_ret)
         rb_obj_hide(power);
         base36_power_cache[base - 2][power_level] = power;
         base36_numdigits_cache[base - 2][power_level] = numdigits;
-        rb_gc_register_mark_object(power);
+        rb_vm_register_global_object(power);
     }
     if (numdigits_ret)
         *numdigits_ret = base36_numdigits_cache[base - 2][power_level];

class.c

@@ -32,6 +32,9 @@
 
 /* Flags of T_CLASS
  *
+ * 0:    RCLASS_IS_ROOT
+ *           The class has been added to the VM roots. Will always be marked and pinned.
+ *           This is done for classes defined from C to allow storing them in global variables.
  * 1:    RUBY_FL_SINGLETON
  *           This class is a singleton class.
  * 2:    RCLASS_SUPERCLASSES_INCLUDE_SELF
@@ -56,6 +59,9 @@
 
 /* Flags of T_MODULE
  *
+ * 0:    RCLASS_IS_ROOT
+ *           The class has been added to the VM roots. Will always be marked and pinned.
+ *           This is done for classes defined from C to allow storing them in global variables.
  * 1:    RMODULE_ALLOCATED_BUT_NOT_INITIALIZED
  *           Module has not been initialized.
  * 2:    RCLASS_SUPERCLASSES_INCLUDE_SELF
@@ -812,7 +818,7 @@ boot_defclass(const char *name, VALUE super)
     ID id = rb_intern(name);
 
     rb_const_set((rb_cObject ? rb_cObject : obj), id, obj);
-    rb_vm_add_root_module(obj);
+    rb_vm_register_global_object(obj);
     return obj;
 }
 
@@ -894,7 +900,7 @@ Init_class_hierarchy(void)
 {
     rb_cBasicObject = boot_defclass("BasicObject", 0);
     rb_cObject = boot_defclass("Object", rb_cBasicObject);
-    rb_gc_register_mark_object(rb_cObject);
+    rb_vm_register_global_object(rb_cObject);
 
     /* resolve class name ASAP for order-independence */
     rb_set_class_path_string(rb_cObject, rb_cObject, rb_fstring_lit("Object"));
@@ -988,14 +994,14 @@ rb_define_class(const char *name, VALUE super)
         }
 
         /* Class may have been defined in Ruby and not pin-rooted */
-        rb_vm_add_root_module(klass);
+        rb_vm_register_global_object(klass);
         return klass;
     }
     if (!super) {
         rb_raise(rb_eArgError, "no super class for '%s'", name);
     }
     klass = rb_define_class_id(id, super);
-    rb_vm_add_root_module(klass);
+    rb_vm_register_global_object(klass);
     rb_const_set(rb_cObject, id, klass);
     rb_class_inherited(super, klass);
 
@@ -1045,7 +1051,7 @@ VALUE
 rb_define_class_id_under(VALUE outer, ID id, VALUE super)
 {
     VALUE klass = rb_define_class_id_under_no_pin(outer, id, super);
-    rb_vm_add_root_module(klass);
+    rb_vm_register_global_object(klass);
     return klass;
 }
 
@@ -1099,11 +1105,11 @@ rb_define_module(const char *name)
                      name, rb_obj_class(module));
         }
         /* Module may have been defined in Ruby and not pin-rooted */
-        rb_vm_add_root_module(module);
+        rb_vm_register_global_object(module);
         return module;
     }
     module = rb_module_new();
-    rb_vm_add_root_module(module);
+    rb_vm_register_global_object(module);
     rb_const_set(rb_cObject, id, module);
 
     return module;
@@ -1128,13 +1134,13 @@ rb_define_module_id_under(VALUE outer, ID id)
                      outer, rb_id2str(id), rb_obj_class(module));
         }
         /* Module may have been defined in Ruby and not pin-rooted */
-        rb_gc_register_mark_object(module);
+        rb_vm_register_global_object(module);
         return module;
     }
     module = rb_module_new();
     rb_const_set(outer, id, module);
     rb_set_class_path_string(module, outer, rb_id2str(id));
-    rb_gc_register_mark_object(module);
+    rb_vm_register_global_object(module);
 
     return module;
 }

complex.c

@@ -2723,7 +2723,7 @@ Init_Complex(void)
                     f_complex_new_bang2(rb_cComplex, ZERO, ONE));
 
 #if !USE_FLONUM
-    rb_gc_register_mark_object(RFLOAT_0 = DBL2NUM(0.0));
+    rb_vm_register_global_object(RFLOAT_0 = DBL2NUM(0.0));
 #endif
 
     rb_provide("complex.so");	/* for backward compatibility */

encoding.c

@@ -1927,7 +1927,7 @@ Init_Encoding(void)
 
     list = rb_encoding_list = rb_ary_new2(ENCODING_LIST_CAPA);
     RBASIC_CLEAR_CLASS(list);
-    rb_gc_register_mark_object(list);
+    rb_vm_register_global_object(list);
 
     for (i = 0; i < enc_table->count; ++i) {
         rb_ary_push(list, enc_new(enc_table->list[i].enc));

enumerator.c

@@ -4562,7 +4562,7 @@ InitVM_Enumerator(void)
     rb_hash_aset(lazy_use_super_method, sym("uniq"), sym("_enumerable_uniq"));
     rb_hash_aset(lazy_use_super_method, sym("with_index"), sym("_enumerable_with_index"));
     rb_obj_freeze(lazy_use_super_method);
-    rb_gc_register_mark_object(lazy_use_super_method);
+    rb_vm_register_global_object(lazy_use_super_method);
 
 #if 0 /* for RDoc */
     rb_define_method(rb_cLazy, "to_a", lazy_to_a, 0);

gc.c

@@ -8767,29 +8767,13 @@ rb_gc_force_recycle(VALUE obj)
     /* no-op */
 }
 
-#ifndef MARK_OBJECT_ARY_BUCKET_SIZE
-#define MARK_OBJECT_ARY_BUCKET_SIZE 1024
-#endif
-
 void
 rb_gc_register_mark_object(VALUE obj)
 {
     if (!is_pointer_to_heap(&rb_objspace, (void *)obj))
         return;
 
-    RB_VM_LOCK_ENTER();
-    {
-        VALUE ary_ary = GET_VM()->mark_object_ary;
-        VALUE ary = rb_ary_last(0, 0, ary_ary);
-
-        if (NIL_P(ary) || RARRAY_LEN(ary) >= MARK_OBJECT_ARY_BUCKET_SIZE) {
-            ary = rb_ary_hidden_new(MARK_OBJECT_ARY_BUCKET_SIZE);
-            rb_ary_push(ary_ary, ary);
-        }
-
-        rb_ary_push(ary, obj);
-    }
-    RB_VM_LOCK_LEAVE();
+    rb_vm_register_global_object(obj);
 }
 
 void

internal/class.h

@@ -108,6 +108,7 @@ struct RClass_and_rb_classext_t {
 #define RCLASS_SUPERCLASSES(c) (RCLASS_EXT(c)->superclasses)
 #define RCLASS_ATTACHED_OBJECT(c) (RCLASS_EXT(c)->as.singleton_class.attached_object)
 
+#define RCLASS_IS_ROOT FL_USER0
 #define RICLASS_IS_ORIGIN FL_USER0
 #define RCLASS_SUPERCLASSES_INCLUDE_SELF FL_USER2
 #define RICLASS_ORIGIN_SHARED_MTBL FL_USER3

internal/vm.h

@@ -45,13 +45,13 @@ VALUE rb_vm_push_frame_fname(struct rb_execution_context_struct *ec, VALUE fname
 /* vm.c */
 VALUE rb_obj_is_thread(VALUE obj);
 void rb_vm_mark(void *ptr);
+void rb_vm_register_global_object(VALUE obj);
 void rb_vm_each_stack_value(void *ptr, void (*cb)(VALUE, void*), void *ctx);
 PUREFUNC(VALUE rb_vm_top_self(void));
 const void **rb_vm_get_insns_address_table(void);
 VALUE rb_source_location(int *pline);
 const char *rb_source_location_cstr(int *pline);
 void rb_vm_pop_cfunc_frame(void);
-int rb_vm_add_root_module(VALUE module);
 void rb_vm_check_redefinition_by_prepend(VALUE klass);
 int rb_vm_check_optimizable_mid(VALUE mid);
 VALUE rb_yield_refine_block(VALUE refinement, VALUE refinements);

io.c

@@ -15629,7 +15629,7 @@ Init_IO(void)
     rb_define_hooked_variable("$,", &rb_output_fs, 0, deprecated_str_setter);
 
     rb_default_rs = rb_fstring_lit("\n"); /* avoid modifying RS_default */
-    rb_gc_register_mark_object(rb_default_rs);
+    rb_vm_register_global_object(rb_default_rs);
     rb_rs = rb_default_rs;
     rb_output_rs = Qnil;
     rb_define_hooked_variable("$/", &rb_rs, 0, deprecated_str_setter);

load.c

@@ -1615,5 +1615,5 @@ Init_load(void)
     rb_define_global_function("autoload?", rb_f_autoload_p, -1);
 
     ruby_dln_libmap = rb_hash_new_with_size(0);
-    rb_gc_register_mark_object(ruby_dln_libmap);
+    rb_vm_register_global_object(ruby_dln_libmap);
 }

marshal.c

@@ -2538,7 +2538,7 @@ compat_allocator_table(void)
 #define RUBY_UNTYPED_DATA_WARNING 0
     compat_allocator_tbl_wrapper =
         Data_Wrap_Struct(0, mark_marshal_compat_t, free_compat_allocator_table, compat_allocator_tbl);
-    rb_gc_register_mark_object(compat_allocator_tbl_wrapper);
+    rb_vm_register_global_object(compat_allocator_tbl_wrapper);
     return compat_allocator_tbl;
 }
 

memory_view.c

@@ -863,7 +863,7 @@ Init_MemoryView(void)
     VALUE obj = TypedData_Wrap_Struct(
         0, &rb_memory_view_exported_object_registry_data_type,
         exported_object_table);
-    rb_gc_register_mark_object(obj);
+    rb_vm_register_global_object(obj);
     rb_memory_view_exported_object_registry = obj;
 
     id_memory_view = rb_intern_const("__memory_view__");

mini_builtin.c

@@ -51,7 +51,7 @@ builtin_iseq_load(const char *feature_name, const struct rb_builtin_function *ta
 
 #ifndef INCLUDED_BY_BUILTIN_C
     st_insert(loaded_builtin_table, (st_data_t)feature_name, (st_data_t)iseq);
-    rb_gc_register_mark_object((VALUE)iseq);
+    rb_vm_register_global_object((VALUE)iseq);
 #endif
 
     return iseq;

numeric.c

@@ -6262,7 +6262,7 @@ Init_Numeric(void)
     rb_fix_to_s_static[8] = rb_fstring_literal("8");
     rb_fix_to_s_static[9] = rb_fstring_literal("9");
     for(int i = 0; i < 10; i++) {
-        rb_gc_register_mark_object(rb_fix_to_s_static[i]);
+        rb_vm_register_global_object(rb_fix_to_s_static[i]);
     }
 
     rb_cFloat  = rb_define_class("Float", rb_cNumeric);

object.c

@@ -517,7 +517,7 @@ rb_obj_clone_setup(VALUE obj, VALUE clone, VALUE kwfreeze)
         static VALUE freeze_true_hash;
         if (!freeze_true_hash) {
             freeze_true_hash = rb_hash_new();
-            rb_gc_register_mark_object(freeze_true_hash);
+            rb_vm_register_global_object(freeze_true_hash);
             rb_hash_aset(freeze_true_hash, ID2SYM(idFreeze), Qtrue);
             rb_obj_freeze(freeze_true_hash);
         }
@@ -541,7 +541,7 @@ rb_obj_clone_setup(VALUE obj, VALUE clone, VALUE kwfreeze)
         static VALUE freeze_false_hash;
         if (!freeze_false_hash) {
             freeze_false_hash = rb_hash_new();
-            rb_gc_register_mark_object(freeze_false_hash);
+            rb_vm_register_global_object(freeze_false_hash);
             rb_hash_aset(freeze_false_hash, ID2SYM(idFreeze), Qfalse);
             rb_obj_freeze(freeze_false_hash);
         }
@@ -4450,7 +4450,7 @@ InitVM_Object(void)
 
     rb_cNilClass = rb_define_class("NilClass", rb_cObject);
     rb_cNilClass_to_s = rb_fstring_enc_lit("", rb_usascii_encoding());
-    rb_gc_register_mark_object(rb_cNilClass_to_s);
+    rb_vm_register_global_object(rb_cNilClass_to_s);
     rb_define_method(rb_cNilClass, "to_s", rb_nil_to_s, 0);
     rb_define_method(rb_cNilClass, "to_a", nil_to_a, 0);
     rb_define_method(rb_cNilClass, "to_h", nil_to_h, 0);
@@ -4536,7 +4536,7 @@ InitVM_Object(void)
 
     rb_cTrueClass = rb_define_class("TrueClass", rb_cObject);
     rb_cTrueClass_to_s = rb_fstring_enc_lit("true", rb_usascii_encoding());
-    rb_gc_register_mark_object(rb_cTrueClass_to_s);
+    rb_vm_register_global_object(rb_cTrueClass_to_s);
     rb_define_method(rb_cTrueClass, "to_s", rb_true_to_s, 0);
     rb_define_alias(rb_cTrueClass, "inspect", "to_s");
     rb_define_method(rb_cTrueClass, "&", true_and, 1);
@@ -4548,7 +4548,7 @@ InitVM_Object(void)
 
     rb_cFalseClass = rb_define_class("FalseClass", rb_cObject);
     rb_cFalseClass_to_s = rb_fstring_enc_lit("false", rb_usascii_encoding());
-    rb_gc_register_mark_object(rb_cFalseClass_to_s);
+    rb_vm_register_global_object(rb_cFalseClass_to_s);
     rb_define_method(rb_cFalseClass, "to_s", rb_false_to_s, 0);
     rb_define_alias(rb_cFalseClass, "inspect", "to_s");
     rb_define_method(rb_cFalseClass, "&", false_and, 1);

proc.c

@@ -1462,7 +1462,7 @@ rb_sym_to_proc(VALUE sym)
 
     if (!sym_proc_cache) {
         sym_proc_cache = rb_ary_hidden_new(SYM_PROC_CACHE_SIZE * 2);
-        rb_gc_register_mark_object(sym_proc_cache);
+        rb_vm_register_global_object(sym_proc_cache);
         rb_ary_store(sym_proc_cache, SYM_PROC_CACHE_SIZE*2 - 1, Qnil);
     }
 

random.c

@@ -596,7 +596,7 @@ fill_random_bytes_crypt(void *seed, size_t size)
             if (prov != INVALID_HCRYPTPROV) {
 #undef RUBY_UNTYPED_DATA_WARNING
 #define RUBY_UNTYPED_DATA_WARNING 0
-                rb_gc_register_mark_object(Data_Wrap_Struct(0, 0, release_crypt, &perm_prov));
+                rb_vm_register_global_object(Data_Wrap_Struct(0, 0, release_crypt, &perm_prov));
             }
         }
         else {			/* another thread acquired */

ruby.c

@@ -2202,7 +2202,7 @@ process_options_global_setup(const ruby_cmdline_options_t *opt, const rb_iseq_t
 
     if ((rb_e_script = opt->e_script) != 0) {
         rb_str_freeze(rb_e_script);
-        rb_gc_register_mark_object(opt->e_script);
+        rb_vm_register_global_object(opt->e_script);
     }
 
     rb_execution_context_t *ec = GET_EC();
@@ -3070,7 +3070,7 @@ ruby_process_options(int argc, char **argv)
     }
     set_progname(external_str_new_cstr(script_name));  /* for the time being */
     rb_argv0 = rb_str_new4(rb_progname);
-    rb_gc_register_mark_object(rb_argv0);
+    rb_vm_register_global_object(rb_argv0);
 
 #ifndef HAVE_SETPROCTITLE
     ruby_init_setproctitle(argc, argv);

struct.c

@@ -496,7 +496,7 @@ rb_struct_define(const char *name, ...)
     }
     else {
         st = new_struct(rb_str_new2(name), rb_cStruct);
-        rb_vm_add_root_module(st);
+        rb_vm_register_global_object(st);
     }
     return setup_struct(st, ary);
 }
@@ -1705,7 +1705,7 @@ rb_data_define(VALUE super, ...)
     va_end(ar);
     if (!super) super = rb_cData;
     VALUE klass = setup_data(anonymous_struct(super), ary);
-    rb_vm_add_root_module(klass);
+    rb_vm_register_global_object(klass);
     return klass;
 }
 

symbol.c

@@ -95,12 +95,12 @@ Init_sym(void)
 
     VALUE dsym_fstrs = rb_ident_hash_new();
     symbols->dsymbol_fstr_hash = dsym_fstrs;
-    rb_gc_register_mark_object(dsym_fstrs);
+    rb_vm_register_global_object(dsym_fstrs);
     rb_obj_hide(dsym_fstrs);
 
     symbols->str_sym = st_init_table_with_size(&symhash, 1000);
     symbols->ids = rb_ary_hidden_new(0);
-    rb_gc_register_mark_object(symbols->ids);
+    rb_vm_register_global_object(symbols->ids);
 
     Init_op_tbl();
     Init_id();

time.c

@@ -1501,7 +1501,7 @@ guess_local_offset(struct vtm *vtm_utc, int *isdst_ret, VALUE *zone_ret)
             localtime_with_gmtoff_zone(&now, &tm, &now_gmtoff, &zone);
             now_isdst = tm.tm_isdst;
             zone = rb_fstring(zone);
-            rb_gc_register_mark_object(zone);
+            rb_vm_register_global_object(zone);
             now_zone = zone;
         }
         if (isdst_ret)
@@ -5773,9 +5773,9 @@ Init_Time(void)
     sym_zone = ID2SYM(rb_intern_const("zone"));
 
     str_utc = rb_fstring_lit("UTC");
-    rb_gc_register_mark_object(str_utc);
+    rb_vm_register_global_object(str_utc);
     str_empty = rb_fstring_lit("");
-    rb_gc_register_mark_object(str_empty);
+    rb_vm_register_global_object(str_empty);
 
     rb_cTime = rb_define_class("Time", rb_cObject);
     VALUE scTime = rb_singleton_class(rb_cTime);

variable.c

@@ -72,11 +72,11 @@ Init_var_tables(void)
 
     autoload_mutex = rb_mutex_new();
     rb_obj_hide(autoload_mutex);
-    rb_gc_register_mark_object(autoload_mutex);
+    rb_vm_register_global_object(autoload_mutex);
 
     autoload_features = rb_ident_hash_new();
     rb_obj_hide(autoload_features);
-    rb_gc_register_mark_object(autoload_features);
+    rb_vm_register_global_object(autoload_features);
 }
 
 static inline bool
@@ -3701,7 +3701,9 @@ rb_define_const(VALUE klass, const char *name, VALUE val)
     if (!rb_is_const_id(id)) {
         rb_warn("rb_define_const: invalid name '%s' for constant", name);
     }
-    rb_gc_register_mark_object(val);
+    if (!RB_SPECIAL_CONST_P(val)) {
+        rb_vm_register_global_object(val);
+    }
     rb_const_set(klass, id, val);
 }