This commit implements the Object Shapes technique in CRuby.

Object Shapes is used for accessing instance variables and representing the
"frozenness" of objects.  Object instances have a "shape" and the shape
represents some attributes of the object (currently which instance variables are
set and the "frozenness").  Shapes form a tree data structure, and when a new
instance variable is set on an object, that object "transitions" to a new shape
in the shape tree.  Each shape has an ID that is used for caching. The shape
structure is independent of class, so objects of different types can have the
same shape.

For example:

```ruby
class Foo
  def initialize
    # Starts with shape id 0
    @A = 1 # transitions to shape id 1
    @b = 1 # transitions to shape id 2
  end
end

class Bar
  def initialize
    # Starts with shape id 0
    @A = 1 # transitions to shape id 1
    @b = 1 # transitions to shape id 2
  end
end

foo = Foo.new # `foo` has shape id 2
bar = Bar.new # `bar` has shape id 2
```

Both `foo` and `bar` instances have the same shape because they both set
instance variables of the same name in the same order.

This technique can help to improve inline cache hits as well as generate more
efficient machine code in JIT compilers.

This commit also adds some methods for debugging shapes on objects.  See
`RubyVM::Shape` for more details.

For more context on Object Shapes, see [Feature: #18776]

Co-Authored-By: Aaron Patterson <tenderlove@ruby-lang.org>
Co-Authored-By: Eileen M. Uchitelle <eileencodes@gmail.com>
Co-Authored-By: John Hawthorn <john@hawthorn.email>

bootstraptest/test_attr.rb

@@ -34,3 +34,19 @@ class A
     print "ok"
   end
 }, '[ruby-core:15120]'
+
+assert_equal %{ok}, %{
+  class Big
+    attr_reader :foo
+    def initialize
+      @foo = "ok"
+    end
+  end
+
+  obj = Big.new
+  100.times do |i|
+    obj.instance_variable_set(:"@ivar_\#{i}", i)
+  end
+
+  Big.new.foo
+}

compile.c

@@ -2058,20 +2058,7 @@ cdhash_set_label_i(VALUE key, VALUE val, VALUE ptr)
 static inline VALUE
 get_ivar_ic_value(rb_iseq_t *iseq,ID id)
 {
-    VALUE val;
-    struct rb_id_table *tbl = ISEQ_COMPILE_DATA(iseq)->ivar_cache_table;
-    if (tbl) {
-        if (rb_id_table_lookup(tbl,id,&val)) {
-            return val;
-        }
-    }
-    else {
-        tbl = rb_id_table_create(1);
-        ISEQ_COMPILE_DATA(iseq)->ivar_cache_table = tbl;
-    }
-    val = INT2FIX(ISEQ_BODY(iseq)->ivc_size++);
-    rb_id_table_insert(tbl,id,val);
-    return val;
+    return INT2FIX(ISEQ_BODY(iseq)->ivc_size++);
 }
 
 static inline VALUE
@@ -2472,9 +2459,13 @@ iseq_set_sequence(rb_iseq_t *iseq, LINK_ANCHOR *const anchor)
                             generated_iseq[code_index + 1 + j] = (VALUE)ic;
                         }
                         break;
+                      case TS_IVC: /* inline ivar cache */
+                        {
+                            unsigned int ic_index = FIX2UINT(operands[j]);
+                            vm_ic_attr_index_initialize(((IVC)&body->is_entries[ic_index]), INVALID_SHAPE_ID);
+                        }
                       case TS_ISE: /* inline storage entry: `once` insn */
                       case TS_ICVARC: /* inline cvar cache */
-                      case TS_IVC: /* inline ivar cache */
                         {
                             unsigned int ic_index = FIX2UINT(operands[j]);
                             IC ic = &ISEQ_IS_ENTRY_START(body, type)[ic_index].ic_cache;
@@ -11514,6 +11505,11 @@ ibf_load_code(const struct ibf_load *load, rb_iseq_t *iseq, ibf_offset_t bytecod
 
                     ISE ic = ISEQ_IS_ENTRY_START(load_body, operand_type) + op;
                     code[code_index] = (VALUE)ic;
+
+                    if (operand_type == TS_IVC) {
+                        vm_ic_attr_index_initialize(((IVC)code[code_index]), INVALID_SHAPE_ID);
+                    }
+
                 }
                 break;
               case TS_CALLDATA:

debug_counter.h

@@ -130,7 +130,6 @@ RB_DEBUG_COUNTER(frame_C2R)
 /* instance variable counts
  *
  * * ivar_get_ic_hit/miss: ivar_get inline cache (ic) hit/miss counts (VM insn)
- * * ivar_get_ic_miss_serial: ivar_get ic miss reason by serial (VM insn)
  * * ivar_get_ic_miss_unset:                      ... by unset (VM insn)
  * * ivar_get_ic_miss_noobject:                   ... by "not T_OBJECT" (VM insn)
  * * ivar_set_...: same counts with ivar_set (VM insn)
@@ -140,17 +139,17 @@ RB_DEBUG_COUNTER(frame_C2R)
  */
 RB_DEBUG_COUNTER(ivar_get_ic_hit)
 RB_DEBUG_COUNTER(ivar_get_ic_miss)
-RB_DEBUG_COUNTER(ivar_get_ic_miss_serial)
-RB_DEBUG_COUNTER(ivar_get_ic_miss_unset)
 RB_DEBUG_COUNTER(ivar_get_ic_miss_noobject)
 RB_DEBUG_COUNTER(ivar_set_ic_hit)
 RB_DEBUG_COUNTER(ivar_set_ic_miss)
-RB_DEBUG_COUNTER(ivar_set_ic_miss_serial)
-RB_DEBUG_COUNTER(ivar_set_ic_miss_unset)
 RB_DEBUG_COUNTER(ivar_set_ic_miss_iv_hit)
 RB_DEBUG_COUNTER(ivar_set_ic_miss_noobject)
 RB_DEBUG_COUNTER(ivar_get_base)
 RB_DEBUG_COUNTER(ivar_set_base)
+RB_DEBUG_COUNTER(ivar_get_ic_miss_set)
+RB_DEBUG_COUNTER(ivar_get_cc_miss_set)
+RB_DEBUG_COUNTER(ivar_get_ic_miss_unset)
+RB_DEBUG_COUNTER(ivar_get_cc_miss_unset)
 
 /* local variable counts
  *

ext/coverage/depend

@@ -165,7 +165,9 @@ coverage.o: $(top_srcdir)/ccan/check_type/check_type.h
 coverage.o: $(top_srcdir)/ccan/container_of/container_of.h
 coverage.o: $(top_srcdir)/ccan/list/list.h
 coverage.o: $(top_srcdir)/ccan/str/str.h
+coverage.o: $(top_srcdir)/constant.h
 coverage.o: $(top_srcdir)/gc.h
+coverage.o: $(top_srcdir)/id_table.h
 coverage.o: $(top_srcdir)/internal.h
 coverage.o: $(top_srcdir)/internal/array.h
 coverage.o: $(top_srcdir)/internal/compilers.h
@@ -176,12 +178,14 @@ coverage.o: $(top_srcdir)/internal/sanitizers.h
 coverage.o: $(top_srcdir)/internal/serial.h
 coverage.o: $(top_srcdir)/internal/static_assert.h
 coverage.o: $(top_srcdir)/internal/thread.h
+coverage.o: $(top_srcdir)/internal/variable.h
 coverage.o: $(top_srcdir)/internal/vm.h
 coverage.o: $(top_srcdir)/internal/warnings.h
 coverage.o: $(top_srcdir)/method.h
 coverage.o: $(top_srcdir)/node.h
 coverage.o: $(top_srcdir)/ruby_assert.h
 coverage.o: $(top_srcdir)/ruby_atomic.h
+coverage.o: $(top_srcdir)/shape.h
 coverage.o: $(top_srcdir)/thread_pthread.h
 coverage.o: $(top_srcdir)/vm_core.h
 coverage.o: $(top_srcdir)/vm_opts.h

ext/objspace/depend

@@ -350,6 +350,7 @@ objspace.o: $(top_srcdir)/internal/serial.h
 objspace.o: $(top_srcdir)/internal/static_assert.h
 objspace.o: $(top_srcdir)/internal/warnings.h
 objspace.o: $(top_srcdir)/node.h
+objspace.o: $(top_srcdir)/shape.h
 objspace.o: $(top_srcdir)/symbol.h
 objspace.o: objspace.c
 objspace.o: {$(VPATH)}id.h
@@ -533,7 +534,9 @@ objspace_dump.o: $(top_srcdir)/ccan/check_type/check_type.h
 objspace_dump.o: $(top_srcdir)/ccan/container_of/container_of.h
 objspace_dump.o: $(top_srcdir)/ccan/list/list.h
 objspace_dump.o: $(top_srcdir)/ccan/str/str.h
+objspace_dump.o: $(top_srcdir)/constant.h
 objspace_dump.o: $(top_srcdir)/gc.h
+objspace_dump.o: $(top_srcdir)/id_table.h
 objspace_dump.o: $(top_srcdir)/internal.h
 objspace_dump.o: $(top_srcdir)/internal/array.h
 objspace_dump.o: $(top_srcdir)/internal/compilers.h
@@ -544,12 +547,14 @@ objspace_dump.o: $(top_srcdir)/internal/sanitizers.h
 objspace_dump.o: $(top_srcdir)/internal/serial.h
 objspace_dump.o: $(top_srcdir)/internal/static_assert.h
 objspace_dump.o: $(top_srcdir)/internal/string.h
+objspace_dump.o: $(top_srcdir)/internal/variable.h
 objspace_dump.o: $(top_srcdir)/internal/vm.h
 objspace_dump.o: $(top_srcdir)/internal/warnings.h
 objspace_dump.o: $(top_srcdir)/method.h
 objspace_dump.o: $(top_srcdir)/node.h
 objspace_dump.o: $(top_srcdir)/ruby_assert.h
 objspace_dump.o: $(top_srcdir)/ruby_atomic.h
+objspace_dump.o: $(top_srcdir)/shape.h
 objspace_dump.o: $(top_srcdir)/thread_pthread.h
 objspace_dump.o: $(top_srcdir)/vm_core.h
 objspace_dump.o: $(top_srcdir)/vm_opts.h

gc.c

@@ -2895,8 +2895,7 @@ rb_class_instance_allocate_internal(VALUE klass, VALUE flags, bool wb_protected)
     GC_ASSERT((flags & RUBY_T_MASK) == T_OBJECT);
     GC_ASSERT(flags & ROBJECT_EMBED);
 
-    st_table *index_tbl = RCLASS_IV_INDEX_TBL(klass);
-    uint32_t index_tbl_num_entries = index_tbl == NULL ? 0 : (uint32_t)index_tbl->num_entries;
+    uint32_t index_tbl_num_entries = RCLASS_EXT(klass)->max_iv_count;
 
     size_t size;
     bool embed = true;
@@ -2931,7 +2930,7 @@ rb_class_instance_allocate_internal(VALUE klass, VALUE flags, bool wb_protected)
 #endif
     }
     else {
-        rb_init_iv_list(obj);
+        rb_ensure_iv_list_size(obj, 0, index_tbl_num_entries);
     }
 
     return obj;
@@ -3206,20 +3205,6 @@ rb_free_const_table(struct rb_id_table *tbl)
     rb_id_table_free(tbl);
 }
 
-static int
-free_iv_index_tbl_free_i(st_data_t key, st_data_t value, st_data_t data)
-{
-    xfree((void *)value);
-    return ST_CONTINUE;
-}
-
-static void
-iv_index_tbl_free(struct st_table *tbl)
-{
-    st_foreach(tbl, free_iv_index_tbl_free_i, 0);
-    st_free_table(tbl);
-}
-
 // alive: if false, target pointers can be freed already.
 //        To check it, we need objspace parameter.
 static void
@@ -3435,6 +3420,16 @@ obj_free(rb_objspace_t *objspace, VALUE obj)
             RB_DEBUG_COUNTER_INC(obj_obj_transient);
         }
         else {
+            rb_shape_t *shape = rb_shape_get_shape_by_id(ROBJECT_SHAPE_ID(obj));
+            if (shape) {
+                VALUE klass = RBASIC_CLASS(obj);
+
+                // Increment max_iv_count if applicable, used to determine size pool allocation
+                uint32_t num_of_ivs = shape->iv_count;
+                if (RCLASS_EXT(klass)->max_iv_count < num_of_ivs) {
+                    RCLASS_EXT(klass)->max_iv_count = num_of_ivs;
+                }
+            }
             xfree(RANY(obj)->as.object.as.heap.ivptr);
             RB_DEBUG_COUNTER_INC(obj_obj_ptr);
         }
@@ -3449,9 +3444,6 @@ obj_free(rb_objspace_t *objspace, VALUE obj)
         if (RCLASS_CONST_TBL(obj)) {
             rb_free_const_table(RCLASS_CONST_TBL(obj));
         }
-        if (RCLASS_IV_INDEX_TBL(obj)) {
-            iv_index_tbl_free(RCLASS_IV_INDEX_TBL(obj));
-        }
         if (RCLASS_CVC_TBL(obj)) {
             rb_id_table_foreach_values(RCLASS_CVC_TBL(obj), cvar_table_free_i, NULL);
             rb_id_table_free(RCLASS_CVC_TBL(obj));
@@ -4873,10 +4865,6 @@ obj_memsize_of(VALUE obj, int use_all_types)
             if (RCLASS_CVC_TBL(obj)) {
                 size += rb_id_table_memsize(RCLASS_CVC_TBL(obj));
             }
-            if (RCLASS_IV_INDEX_TBL(obj)) {
-                // TODO: more correct value
-                size += st_memsize(RCLASS_IV_INDEX_TBL(obj));
-            }
             if (RCLASS_EXT(obj)->iv_tbl) {
                 size += st_memsize(RCLASS_EXT(obj)->iv_tbl);
             }
@@ -10407,27 +10395,13 @@ update_subclass_entries(rb_objspace_t *objspace, rb_subclass_entry_t *entry)
     }
 }
 
-static int
-update_iv_index_tbl_i(st_data_t key, st_data_t value, st_data_t arg)
-{
-    rb_objspace_t *objspace = (rb_objspace_t *)arg;
-    struct rb_iv_index_tbl_entry *ent = (struct rb_iv_index_tbl_entry *)value;
-    UPDATE_IF_MOVED(objspace, ent->class_value);
-    return ST_CONTINUE;
-}
-
 static void
 update_class_ext(rb_objspace_t *objspace, rb_classext_t *ext)
 {
     UPDATE_IF_MOVED(objspace, ext->origin_);
     UPDATE_IF_MOVED(objspace, ext->includer);
     UPDATE_IF_MOVED(objspace, ext->refined_class);
     update_subclass_entries(objspace, ext->subclasses);
-
-    // ext->iv_index_tbl
-    if (ext->iv_index_tbl) {
-        st_foreach(ext->iv_index_tbl, update_iv_index_tbl_i, (st_data_t)objspace);
-    }
 }
 
 static void

include/ruby/internal/core/robject.h

@@ -46,7 +46,6 @@
 #define ROBJECT_EMBED         ROBJECT_EMBED
 #define ROBJECT_NUMIV         ROBJECT_NUMIV
 #define ROBJECT_IVPTR         ROBJECT_IVPTR
-#define ROBJECT_IV_INDEX_TBL  ROBJECT_IV_INDEX_TBL
 /** @endcond */
 
 /**
@@ -132,7 +131,7 @@ struct RObject {
              *
              * This is a shortcut for `RCLASS_IV_INDEX_TBL(rb_obj_class(obj))`.
              */
-            struct st_table *iv_index_tbl;
+            struct rb_id_table *iv_index_tbl;
         } heap;
 
 #if USE_RVARGC

include/ruby/internal/fl_type.h

@@ -941,21 +941,8 @@ RB_OBJ_FREEZE_RAW(VALUE obj)
     RB_FL_SET_RAW(obj, RUBY_FL_FREEZE);
 }
 
-/**
- * Prevents further modifications to the given object.  ::rb_eFrozenError shall
- * be raised if modification is attempted.
- *
- * @param[out]  x  Object in question.
- */
-static inline void
-rb_obj_freeze_inline(VALUE x)
-{
-    if (RB_FL_ABLE(x)) {
-        RB_OBJ_FREEZE_RAW(x);
-        if (RBASIC_CLASS(x) && !(RBASIC(x)->flags & RUBY_FL_SINGLETON)) {
-            rb_freeze_singleton_class(x);
-        }
-    }
-}
+RUBY_SYMBOL_EXPORT_BEGIN
+void rb_obj_freeze_inline(VALUE obj);
+RUBY_SYMBOL_EXPORT_END
 
 #endif /* RBIMPL_FL_TYPE_H */

inits.c

@@ -77,6 +77,7 @@ rb_call_inits(void)
     CALL(vm_stack_canary);
     CALL(ast);
     CALL(gc_stress);
+    CALL(shape);
 
     // enable builtin loading
     CALL(builtin);

internal.h

@@ -48,9 +48,6 @@
 #undef RHASH_TBL
 #undef RHASH_EMPTY_P
 
-/* internal/object.h */
-#undef ROBJECT_IV_INDEX_TBL
-
 /* internal/struct.h */
 #undef RSTRUCT_LEN
 #undef RSTRUCT_PTR

internal/class.h

@@ -14,6 +14,7 @@
 #include "ruby/internal/stdbool.h"     /* for bool */
 #include "ruby/intern.h"        /* for rb_alloc_func_t */
 #include "ruby/ruby.h"          /* for struct RBasic */
+#include "shape.h"
 
 #ifdef RCLASS_SUPER
 # undef RCLASS_SUPER
@@ -27,8 +28,8 @@ struct rb_subclass_entry {
 
 struct rb_iv_index_tbl_entry {
     uint32_t index;
-    rb_serial_t class_serial;
-    VALUE class_value;
+    shape_id_t source_shape_id;
+    shape_id_t dest_shape_id;
 };
 
 struct rb_cvar_class_tbl_entry {
@@ -38,7 +39,6 @@ struct rb_cvar_class_tbl_entry {
 };
 
 struct rb_classext_struct {
-    struct st_table *iv_index_tbl; // ID -> struct rb_iv_index_tbl_entry
     struct st_table *iv_tbl;
 #if SIZEOF_SERIAL_T == SIZEOF_VALUE /* otherwise m_tbl is in struct RClass */
     struct rb_id_table *m_tbl;
@@ -64,6 +64,10 @@ struct rb_classext_struct {
     const VALUE refined_class;
     rb_alloc_func_t allocator;
     const VALUE includer;
+    uint32_t max_iv_count;
+#if !SHAPE_IN_BASIC_FLAGS
+    shape_id_t shape_id;
+#endif
 };
 
 struct RClass {
@@ -102,7 +106,6 @@ typedef struct rb_classext_struct rb_classext_t;
 #define RCLASS_CALLABLE_M_TBL(c) (RCLASS_EXT(c)->callable_m_tbl)
 #define RCLASS_CC_TBL(c) (RCLASS_EXT(c)->cc_tbl)
 #define RCLASS_CVC_TBL(c) (RCLASS_EXT(c)->cvc_tbl)
-#define RCLASS_IV_INDEX_TBL(c) (RCLASS_EXT(c)->iv_index_tbl)
 #define RCLASS_ORIGIN(c) (RCLASS_EXT(c)->origin_)
 #define RCLASS_REFINED_CLASS(c) (RCLASS_EXT(c)->refined_class)
 #if SIZEOF_SERIAL_T == SIZEOF_VALUE