Add a cache for class variables

This change implements a cache for class variables. Previously there was
no cache for cvars. Cvar access is slow due to needing to travel all the
way up th ancestor tree before returning the cvar value. The deeper the
ancestor tree the slower cvar access will be.

The benefits of the cache are more visible with a higher number of
included modules due to the way Ruby looks up class variables. The
benchmark here includes 26 modules and shows with the cache, this branch
is 6.5x faster when accessing class variables.

```
compare-ruby: ruby 3.1.0dev (2021-03-15T06:22:34Z master 9e5105c) [x86_64-darwin19]
built-ruby: ruby 3.1.0dev (2021-03-15T12:12:44Z add-cache-for-clas.. c6be0093ae) [x86_64-darwin19]

|         |compare-ruby|built-ruby|
|:--------|-----------:|---------:|
|vm_cvar  |      5.681M|   36.980M|
|         |           -|     6.51x|
```

Benchmark.ips calling `ActiveRecord::Base.logger` from within a Rails
application. ActiveRecord::Base.logger has 71 ancestors. The more
ancestors a tree has, the more clear the speed increase. IE if Base had
only one ancestor we'd see no improvement. This benchmark is run on a
vanilla Rails application.

Benchmark code:

```ruby
require "benchmark/ips"
require_relative "config/environment"

Benchmark.ips do |x|
  x.report "logger" do
    ActiveRecord::Base.logger
  end
end
```

Ruby 3.0 master / Rails 6.1:

```
Warming up --------------------------------------
              logger   155.251k i/100ms
Calculating -------------------------------------
```

Ruby 3.0 with cvar cache /  Rails 6.1:

```
Warming up --------------------------------------
              logger     1.546M i/100ms
Calculating -------------------------------------
              logger     14.857M (± 4.8%) i/s -     74.198M in   5.006202s
```

Lastly we ran a benchmark to demonstate the difference between master
and our cache when the number of modules increases. This benchmark
measures 1 ancestor, 30 ancestors, and 100 ancestors.

Ruby 3.0 master:

```
Warming up --------------------------------------
            1 module     1.231M i/100ms
          30 modules   432.020k i/100ms
         100 modules   145.399k i/100ms
Calculating -------------------------------------
            1 module     12.210M (± 2.1%) i/s -     61.553M in   5.043400s
          30 modules      4.354M (± 2.7%) i/s -     22.033M in   5.063839s
         100 modules      1.434M (± 2.9%) i/s -      7.270M in   5.072531s

Comparison:
            1 module: 12209958.3 i/s
          30 modules:  4354217.8 i/s - 2.80x  (± 0.00) slower
         100 modules:  1434447.3 i/s - 8.51x  (± 0.00) slower
```

Ruby 3.0 with cvar cache:

```
Warming up --------------------------------------
            1 module     1.641M i/100ms
          30 modules     1.655M i/100ms
         100 modules     1.620M i/100ms
Calculating -------------------------------------
            1 module     16.279M (± 3.8%) i/s -     82.038M in   5.046923s
          30 modules     15.891M (± 3.9%) i/s -     79.459M in   5.007958s
         100 modules     16.087M (± 3.6%) i/s -     81.005M in   5.041931s

Comparison:
            1 module: 16279458.0 i/s
         100 modules: 16087484.6 i/s - same-ish: difference falls within error
          30 modules: 15891406.2 i/s - same-ish: difference falls within error
```

Co-authored-by: Aaron Patterson <tenderlove@ruby-lang.org>

benchmark/vm_cvar.yml

@@ -0,0 +1,20 @@
+prelude: |
+  class A
+    @@foo = 1
+
+    def self.foo
+      @@foo
+    end
+
+    ("A".."Z").each do |module_name|
+      eval <<-EOM
+        module #{module_name}
+        end
+
+        include #{module_name}
+      EOM
+    end
+  end
+benchmark:
+  vm_cvar: A.foo
+loop_count: 600000

class.c

@@ -27,6 +27,7 @@
 #include <ctype.h>
 
 #include "constant.h"
+#include "debug_counter.h"
 #include "id_table.h"
 #include "internal.h"
 #include "internal/class.h"
@@ -43,6 +44,8 @@
 #define METACLASS_OF(k) RBASIC(k)->klass
 #define SET_METACLASS_OF(k, cls) RBASIC_SET_CLASS(k, cls)
 
+RUBY_EXTERN rb_serial_t ruby_vm_global_cvar_state;
+
 void
 rb_class_subclass_add(VALUE super, VALUE klass)
 {
@@ -1085,6 +1088,8 @@ do_include_modules_at(const VALUE klass, VALUE c, VALUE module, int search_super
         VALUE super_class = RCLASS_SUPER(c);
 
         // invalidate inline method cache
+        RB_DEBUG_COUNTER_INC(cvar_include_invalidate);
+        ruby_vm_global_cvar_state++;
         tbl = RCLASS_M_TBL(module);
         if (tbl && rb_id_table_size(tbl)) {
             if (search_super) { // include

common.mk

@@ -2458,6 +2458,7 @@ class.$(OBJEXT): {$(VPATH)}backward/2/stdarg.h
 class.$(OBJEXT): {$(VPATH)}class.c
 class.$(OBJEXT): {$(VPATH)}config.h
 class.$(OBJEXT): {$(VPATH)}constant.h
+class.$(OBJEXT): {$(VPATH)}debug_counter.h
 class.$(OBJEXT): {$(VPATH)}defines.h
 class.$(OBJEXT): {$(VPATH)}encoding.h
 class.$(OBJEXT): {$(VPATH)}id.h

compile.c

@@ -8668,8 +8668,9 @@ iseq_compile_each0(rb_iseq_t *iseq, LINK_ANCHOR *const ret, const NODE *node, in
       }
       case NODE_CVAR:{
 	if (!popped) {
-	    ADD_INSN1(ret, line_node, getclassvariable,
-		      ID2SYM(node->nd_vid));
+	    ADD_INSN2(ret, line_node, getclassvariable,
+		      ID2SYM(node->nd_vid),
+		      get_ivar_ic_value(iseq,node->nd_vid));
 	}
 	break;
       }

debug_counter.h

@@ -24,6 +24,11 @@ RB_DEBUG_COUNTER(mc_inline_miss_same_cme)    // IMC miss, but same CME
 RB_DEBUG_COUNTER(mc_inline_miss_same_def)    // IMC miss, but same definition
 RB_DEBUG_COUNTER(mc_inline_miss_diff)        // IMC miss, different methods
 
+RB_DEBUG_COUNTER(cvar_inline_hit)            // cvar cache hit
+RB_DEBUG_COUNTER(cvar_inline_miss)           // miss inline cache
+RB_DEBUG_COUNTER(cvar_class_invalidate)      // invalidate cvar cache when define a cvar that's defined on a subclass
+RB_DEBUG_COUNTER(cvar_include_invalidate)    // invalidate cvar cache on module include or prepend
+
 RB_DEBUG_COUNTER(mc_cme_complement)          // number of acquiring complement CME
 RB_DEBUG_COUNTER(mc_cme_complement_hit)      // number of cache hit for complemented CME
 

gc.c

@@ -3003,6 +3003,13 @@ cc_table_free(rb_objspace_t *objspace, VALUE klass, bool alive)
     }
 }
 
+static enum rb_id_table_iterator_result
+cvar_table_free_i(VALUE value, void * ctx)
+{
+    xfree((void *) value);
+    return ID_TABLE_CONTINUE;
+}
+
 void
 rb_cc_table_free(VALUE klass)
 {
@@ -3114,6 +3121,10 @@ obj_free(rb_objspace_t *objspace, VALUE obj)
 	if (RCLASS_IV_INDEX_TBL(obj)) {
             iv_index_tbl_free(RCLASS_IV_INDEX_TBL(obj));
 	}
+	if (RCLASS_CVC_TBL(obj)) {
+            rb_id_table_foreach_values(RCLASS_CVC_TBL(obj), cvar_table_free_i, NULL);
+            rb_id_table_free(RCLASS_CVC_TBL(obj));
+	}
 	if (RCLASS_SUBCLASSES(obj)) {
 	    if (BUILTIN_TYPE(obj) == T_MODULE) {
 		rb_class_detach_module_subclasses(obj);
@@ -4557,6 +4568,9 @@ obj_memsize_of(VALUE obj, int use_all_types)
 	    if (RCLASS_IV_TBL(obj)) {
 		size += st_memsize(RCLASS_IV_TBL(obj));
 	    }
+	    if (RCLASS_CVC_TBL(obj)) {
+		size += rb_id_table_memsize(RCLASS_CVC_TBL(obj));
+	    }
 	    if (RCLASS_IV_INDEX_TBL(obj)) {
                 // TODO: more correct value
 		size += st_memsize(RCLASS_IV_INDEX_TBL(obj));
@@ -9603,6 +9617,27 @@ update_cc_tbl(rb_objspace_t *objspace, VALUE klass)
     }
 }
 
+static enum rb_id_table_iterator_result
+update_cvc_tbl_i(ID id, VALUE cvc_entry, void *data)
+{
+    struct rb_cvar_class_tbl_entry *entry;
+
+    entry = (struct rb_cvar_class_tbl_entry *)cvc_entry;
+
+    entry->class_value = rb_gc_location(entry->class_value);
+
+    return ID_TABLE_CONTINUE;
+}
+
+static void
+update_cvc_tbl(rb_objspace_t *objspace, VALUE klass)
+{
+    struct rb_id_table *tbl = RCLASS_CVC_TBL(klass);
+    if (tbl) {
+        rb_id_table_foreach_with_replace(tbl, update_cvc_tbl_i, 0, objspace);
+    }
+}
+
 static enum rb_id_table_iterator_result
 update_const_table(VALUE value, void *data)
 {
@@ -9674,6 +9709,7 @@ gc_update_object_references(rb_objspace_t *objspace, VALUE obj)
         if (!RCLASS_EXT(obj)) break;
         update_m_tbl(objspace, RCLASS_M_TBL(obj));
         update_cc_tbl(objspace, obj);
+        update_cvc_tbl(objspace, obj);
 
         gc_update_tbl_refs(objspace, RCLASS_IV_TBL(obj));
 

id_table.c

@@ -92,7 +92,7 @@ rb_id_table_init(struct rb_id_table *tbl, int capa)
     return tbl;
 }
 
-struct rb_id_table *
+MJIT_FUNC_EXPORTED struct rb_id_table *
 rb_id_table_create(size_t capa)
 {
     struct rb_id_table *tbl = ALLOC(struct rb_id_table);
@@ -223,7 +223,7 @@ hash_table_show(struct rb_id_table *tbl)
 }
 #endif
 
-int
+MJIT_FUNC_EXPORTED int
 rb_id_table_lookup(struct rb_id_table *tbl, ID id, VALUE *valp)
 {
     id_key_t key = id2key(id);
@@ -253,7 +253,7 @@ rb_id_table_insert_key(struct rb_id_table *tbl, const id_key_t key, const VALUE
     return TRUE;
 }
 
-int
+MJIT_FUNC_EXPORTED int
 rb_id_table_insert(struct rb_id_table *tbl, ID id, VALUE val)
 {
     return rb_id_table_insert_key(tbl, id2key(id), val);

include/ruby/internal/intern/variable.h

@@ -72,6 +72,7 @@ VALUE rb_mod_const_missing(VALUE,VALUE);
 VALUE rb_cvar_defined(VALUE, ID);
 void rb_cvar_set(VALUE, ID, VALUE);
 VALUE rb_cvar_get(VALUE, ID);
+VALUE rb_cvar_find(VALUE, ID, VALUE*);
 void rb_cv_set(VALUE, const char*, VALUE);
 VALUE rb_cv_get(VALUE, const char*);
 void rb_define_class_variable(VALUE, const char*, VALUE);

insns.def

@@ -230,13 +230,15 @@ setinstancevariable
 /* Get value of class variable id of klass as val. */
 DEFINE_INSN
 getclassvariable
-(ID id)
+(ID id, IVC ic)
 ()
 (VALUE val)
 /* "class variable access from toplevel" warning can be hooked. */
 // attr bool leaf = false; /* has rb_warning() */
 {
-    val = rb_cvar_get(vm_get_cvar_base(vm_get_cref(GET_EP()), GET_CFP(), 1), id);
+    rb_cref_t * cref = vm_get_cref(GET_EP());
+    rb_control_frame_t *cfp = GET_CFP();
+    val = vm_getclassvariable(GET_ISEQ(), cref, cfp, id, (ICVARC)ic);
 }
 
 /* Set value of class variable id of klass as val. */
@@ -249,7 +251,7 @@ setclassvariable
 // attr bool leaf = false; /* has rb_warning() */
 {
     vm_ensure_not_refinement_module(GET_SELF());
-    rb_cvar_set(vm_get_cvar_base(vm_get_cref(GET_EP()), GET_CFP(), 1), id, val);
+    vm_setclassvariable(vm_get_cref(GET_EP()), GET_CFP(), id,  val);
 }
 
 /* Get constant variable id. If klass is Qnil and allow_nil is Qtrue, constants

internal/class.h

@@ -31,6 +31,12 @@ struct rb_iv_index_tbl_entry {
     VALUE class_value;
 };
 
+struct rb_cvar_class_tbl_entry {
+    uint32_t index;
+    rb_serial_t global_cvar_state;
+    VALUE class_value;
+};
+
 struct rb_classext_struct {
     struct st_table *iv_index_tbl; // ID -> struct rb_iv_index_tbl_entry
     struct st_table *iv_tbl;
@@ -40,6 +46,7 @@ struct rb_classext_struct {
     struct rb_id_table *const_tbl;
     struct rb_id_table *callable_m_tbl;
     struct rb_id_table *cc_tbl; /* ID -> [[ci, cc1], cc2, ...] */
+    struct rb_id_table *cvc_tbl;
     struct rb_subclass_entry *subclasses;
     struct rb_subclass_entry **parent_subclasses;
     /**
@@ -83,6 +90,7 @@ typedef struct rb_classext_struct rb_classext_t;
 #endif
 #define RCLASS_CALLABLE_M_TBL(c) (RCLASS_EXT(c)->callable_m_tbl)
 #define RCLASS_CC_TBL(c) (RCLASS_EXT(c)->cc_tbl)
+#define RCLASS_CVC_TBL(c) (RCLASS_EXT(c)->cvc_tbl)
 #define RCLASS_IV_INDEX_TBL(c) (RCLASS_EXT(c)->iv_index_tbl)
 #define RCLASS_ORIGIN(c) (RCLASS_EXT(c)->origin_)
 #define RCLASS_REFINED_CLASS(c) (RCLASS_EXT(c)->refined_class)

variable.c

@@ -39,6 +39,8 @@
 #include "ractor_core.h"
 #include "vm_sync.h"
 
+RUBY_EXTERN rb_serial_t ruby_vm_global_cvar_state;
+
 typedef void rb_gvar_compact_t(void *var);
 
 static struct rb_id_table *rb_global_tbl;
@@ -3325,6 +3327,30 @@ cvar_overtaken(VALUE front, VALUE target, ID id)
     }
 }
 
+static VALUE
+find_cvar(VALUE klass, VALUE * front, VALUE * target, ID id)
+{
+    VALUE v = Qundef;
+    CVAR_ACCESSOR_SHOULD_BE_MAIN_RACTOR();
+    if (cvar_lookup_at(klass, id, (&v))) {
+        if (!*front) {
+            *front = klass;
+        }
+        *target = klass;
+    }
+
+    for (klass = cvar_front_klass(klass); klass; klass = RCLASS_SUPER(klass)) {
+	if (cvar_lookup_at(klass, id, (&v))) {
+            if (!*front) {
+                *front = klass;
+            }
+            *target = klass;
+        }
+    }
+
+    return v;
+}
+
 #define CVAR_FOREACH_ANCESTORS(klass, v, r) \
     for (klass = cvar_front_klass(klass); klass; klass = RCLASS_SUPER(klass)) { \
 	if (cvar_lookup_at(klass, id, (v))) { \
@@ -3338,6 +3364,20 @@ cvar_overtaken(VALUE front, VALUE target, ID id)
     CVAR_FOREACH_ANCESTORS(klass, v, r);\
 } while(0)
 
+static void
+check_for_cvar_table(VALUE subclass, VALUE key)
+{
+    st_table *tbl = RCLASS_IV_TBL(subclass);
+
+    if (tbl && st_lookup(tbl, key, NULL)) {
+        RB_DEBUG_COUNTER_INC(cvar_class_invalidate);
+        ruby_vm_global_cvar_state++;
+        return;
+    }
+
+    rb_class_foreach_subclass(subclass, check_for_cvar_table, key);
+}
+
 void
 rb_cvar_set(VALUE klass, ID id, VALUE val)
 {
@@ -3357,25 +3397,42 @@ rb_cvar_set(VALUE klass, ID id, VALUE val)
     }
     check_before_mod_set(target, id, val, "class variable");
 
-    rb_class_ivar_set(target, id, val);
+    int result = rb_class_ivar_set(target, id, val);
+
+    // Break the cvar cache if this is a new class variable
+    // and target is a module or a subclass with the same
+    // cvar in this lookup.
+    if (result == 0) {
+        if (RB_TYPE_P(target, T_CLASS)) {
+            if (RCLASS_SUBCLASSES(target)) {
+                rb_class_foreach_subclass(target, check_for_cvar_table, id);
+            }
+        }
+    }
 }
 
 VALUE
-rb_cvar_get(VALUE klass, ID id)
+rb_cvar_find(VALUE klass, ID id, VALUE *front)
 {
-    VALUE tmp, front = 0, target = 0;
-    st_data_t value;
+    VALUE target = 0;
+    VALUE value;
 
-    tmp = klass;
-    CVAR_LOOKUP(&value, {if (!front) front = klass; target = klass;});
+    value = find_cvar(klass, front, &target, id);
     if (!target) {
 	rb_name_err_raise("uninitialized class variable %1$s in %2$s",
-			  tmp, ID2SYM(id));
+			  klass, ID2SYM(id));
     }
-    cvar_overtaken(front, target, id);
+    cvar_overtaken(*front, target, id);
     return (VALUE)value;
 }
 
+VALUE
+rb_cvar_get(VALUE klass, ID id)
+{
+    VALUE front = 0;
+    return rb_cvar_find(klass, id, &front);
+}
+
 VALUE
 rb_cvar_defined(VALUE klass, ID id)
 {

vm.c

@@ -405,6 +405,7 @@ unsigned int    ruby_vm_event_local_num;
 
 rb_serial_t ruby_vm_global_constant_state = 1;
 rb_serial_t ruby_vm_class_serial = 1;
+rb_serial_t ruby_vm_global_cvar_state = 1;
 
 static const struct rb_callcache vm_empty_cc = {
     .flags = T_IMEMO | (imemo_callcache << FL_USHIFT) | VM_CALLCACHE_UNMARKABLE,
@@ -484,7 +485,7 @@ rb_dtrace_setup(rb_execution_context_t *ec, VALUE klass, ID id,
 static VALUE
 vm_stat(int argc, VALUE *argv, VALUE self)
 {
-    static VALUE sym_global_constant_state, sym_class_serial;
+    static VALUE sym_global_constant_state, sym_class_serial, sym_global_cvar_state;
     VALUE arg = Qnil;
     VALUE hash = Qnil, key = Qnil;
 
@@ -505,6 +506,7 @@ vm_stat(int argc, VALUE *argv, VALUE self)
 #define S(s) sym_##s = ID2SYM(rb_intern_const(#s))
 	S(global_constant_state);
 	S(class_serial);
+	S(global_cvar_state);
 #undef S
     }
 
@@ -516,6 +518,7 @@ vm_stat(int argc, VALUE *argv, VALUE self)
 
     SET(global_constant_state, ruby_vm_global_constant_state);
     SET(class_serial, ruby_vm_class_serial);
+    SET(global_cvar_state, ruby_vm_global_cvar_state);
 #undef SET
 
     if (!NIL_P(key)) { /* matched key should return above */