YJIT: Introduce no_gc attribute

bootstraptest/test_yjit.rb

@@ -3654,3 +3654,8 @@ def custom.to_a
   test(custom)
   EMPTY
 }
+
+# Test leaf no_gc builtin function
+assert_equal '97', %q{
+  "a".getbyte(0)
+}

common.mk

@@ -1066,6 +1066,7 @@ BUILTIN_RB_SRCS = \
 		$(srcdir)/array.rb \
 		$(srcdir)/kernel.rb \
 		$(srcdir)/ractor.rb \
+		$(srcdir)/string.rb \
 		$(srcdir)/symbol.rb \
 		$(srcdir)/timev.rb \
 		$(srcdir)/thread_sync.rb \
@@ -14985,6 +14986,7 @@ string.$(OBJEXT): $(top_srcdir)/internal/vm.h
 string.$(OBJEXT): $(top_srcdir)/internal/warnings.h
 string.$(OBJEXT): {$(VPATH)}assert.h
 string.$(OBJEXT): {$(VPATH)}atomic.h
+string.$(OBJEXT): {$(VPATH)}builtin.h
 string.$(OBJEXT): {$(VPATH)}backward/2/assume.h
 string.$(OBJEXT): {$(VPATH)}backward/2/attributes.h
 string.$(OBJEXT): {$(VPATH)}backward/2/bool.h
@@ -15164,6 +15166,7 @@ string.$(OBJEXT): {$(VPATH)}ruby_assert.h
 string.$(OBJEXT): {$(VPATH)}shape.h
 string.$(OBJEXT): {$(VPATH)}st.h
 string.$(OBJEXT): {$(VPATH)}string.c
+string.$(OBJEXT): {$(VPATH)}string.rbinc
 string.$(OBJEXT): {$(VPATH)}subst.h
 string.$(OBJEXT): {$(VPATH)}thread_native.h
 string.$(OBJEXT): {$(VPATH)}util.h

compile.c

@@ -8236,6 +8236,9 @@ compile_builtin_attr(rb_iseq_t *iseq, const NODE *node)
         if (strcmp(RSTRING_PTR(string), "leaf") == 0) {
             ISEQ_BODY(iseq)->builtin_attrs |= BUILTIN_ATTR_LEAF;
         }
+        else if (strcmp(RSTRING_PTR(string), "no_gc") == 0) {
+            ISEQ_BODY(iseq)->builtin_attrs |= BUILTIN_ATTR_NO_GC;
+        }
         else {
             goto unknown_arg;
         }

inits.c

@@ -100,6 +100,7 @@ rb_call_builtin_inits(void)
     BUILTIN(warning);
     BUILTIN(array);
     BUILTIN(kernel);
+    BUILTIN(string);
     BUILTIN(symbol);
     BUILTIN(timev);
     BUILTIN(thread_sync);

string.c

@@ -24,6 +24,7 @@
 #include "debug_counter.h"
 #include "encindex.h"
 #include "id.h"
+#include "builtin.h"
 #include "internal.h"
 #include "internal/array.h"
 #include "internal/compar.h"
@@ -6085,6 +6086,12 @@ rb_str_getbyte(VALUE str, VALUE index)
     return INT2FIX((unsigned char)RSTRING_PTR(str)[pos]);
 }
 
+static VALUE
+str_getbyte(rb_execution_context_t *ec, VALUE self, VALUE index)
+{
+    return rb_str_getbyte(self, index);
+}
+
 /*
  *  call-seq:
  *    setbyte(index, integer) -> integer
@@ -12003,7 +12010,6 @@ Init_String(void)
     rb_define_method(rb_cString, "replace", rb_str_replace, 1);
     rb_define_method(rb_cString, "clear", rb_str_clear, 0);
     rb_define_method(rb_cString, "chr", rb_str_chr, 0);
-    rb_define_method(rb_cString, "getbyte", rb_str_getbyte, 1);
     rb_define_method(rb_cString, "setbyte", rb_str_setbyte, 2);
     rb_define_method(rb_cString, "byteslice", rb_str_byteslice, -1);
     rb_define_method(rb_cString, "bytesplice", rb_str_bytesplice, -1);
@@ -12169,3 +12175,5 @@ Init_String(void)
 
     rb_define_method(rb_cSymbol, "encoding", sym_encoding, 0);
 }
+
+#include "string.rbinc"

string.rb

@@ -549,4 +549,20 @@
 #    as determined by a given record separator.
 #  - #upto: Calls the given block with each string value returned by successive calls to #succ.
 
-class String; end
+class String
+  # call-seq:
+  #   getbyte(index) -> integer or nil
+  #
+  # Returns the byte at zero-based +index+ as an integer, or +nil+ if +index+ is out of range:
+  #
+  #   s = 'abcde'   # => "abcde"
+  #   s.getbyte(0)  # => 97
+  #   s.getbyte(-1) # => 101
+  #   s.getbyte(5)  # => nil
+  #
+  # Related: String#setbyte.
+  def getbyte(index)
+    Primitive.attr! :leaf, :no_gc
+    Primitive.str_getbyte(index)
+  end
+end

tool/mk_builtin_loader.rb

@@ -6,7 +6,7 @@
 
 SUBLIBS = {}
 REQUIRED = {}
-BUILTIN_ATTRS = %w[leaf]
+BUILTIN_ATTRS = %w[leaf no_gc]
 
 def string_literal(lit, str = [])
   while lit

vm_core.h

@@ -367,8 +367,10 @@ enum rb_iseq_type {
 
 // Attributes specified by Primitive.attr!
 enum rb_builtin_attr {
-    // If true, this ISeq does not call methods.
+    // The iseq does not call methods.
     BUILTIN_ATTR_LEAF = 0x01,
+    // The iseq does not allocate objects.
+    BUILTIN_ATTR_NO_GC = 0x02,
 };
 
 struct rb_iseq_constant_body {

yjit.c

@@ -724,28 +724,33 @@ rb_optimized_call(VALUE *recv, rb_execution_context_t *ec, int argc, VALUE *argv
     return rb_vm_invoke_proc(ec, proc, argc, argv, kw_splat, block_handler);
 }
 
+unsigned int
+rb_yjit_iseq_builtin_attrs(const rb_iseq_t *iseq)
+{
+    return iseq->body->builtin_attrs;
+}
 
-// If true, the iseq is leaf and it can be replaced by a single C call.
-bool
-rb_leaf_invokebuiltin_iseq_p(const rb_iseq_t *iseq)
+// If true, the iseq has only opt_invokebuiltin_delegate_leave and leave insns.
+static bool
+invokebuiltin_delegate_leave_p(const rb_iseq_t *iseq)
 {
     unsigned int invokebuiltin_len = insn_len(BIN(opt_invokebuiltin_delegate_leave));
     unsigned int leave_len = insn_len(BIN(leave));
-
-    return (iseq->body->iseq_size == (invokebuiltin_len + leave_len) &&
+    return iseq->body->iseq_size == (invokebuiltin_len + leave_len) &&
         rb_vm_insn_addr2opcode((void *)iseq->body->iseq_encoded[0]) == BIN(opt_invokebuiltin_delegate_leave) &&
-        rb_vm_insn_addr2opcode((void *)iseq->body->iseq_encoded[invokebuiltin_len]) == BIN(leave) &&
-        (iseq->body->builtin_attrs & BUILTIN_ATTR_LEAF) != 0
-    );
+        rb_vm_insn_addr2opcode((void *)iseq->body->iseq_encoded[invokebuiltin_len]) == BIN(leave);
 }
 
-// Return an rb_builtin_function if the iseq contains only that leaf builtin function.
+// Return an rb_builtin_function if the iseq contains only that builtin function.
 const struct rb_builtin_function *
-rb_leaf_builtin_function(const rb_iseq_t *iseq)
+rb_yjit_builtin_function(const rb_iseq_t *iseq)
 {
-    if (!rb_leaf_invokebuiltin_iseq_p(iseq))
+    if (invokebuiltin_delegate_leave_p(iseq)) {
+        return (const struct rb_builtin_function *)iseq->body->iseq_encoded[1];
+    }
+    else {
         return NULL;
-    return (const struct rb_builtin_function *)iseq->body->iseq_encoded[1];
+    }
 }
 
 VALUE

yjit/bindgen/src/main.rs

@@ -286,6 +286,7 @@ fn main() {
         .allowlist_var("VM_ENV_DATA_INDEX_FLAGS")
         .allowlist_var("VM_ENV_DATA_SIZE")
         .allowlist_function("rb_iseq_path")
+        .allowlist_type("rb_builtin_attr")
 
         // From yjit.c
         .allowlist_function("rb_iseq_(get|set)_yjit_payload")
@@ -296,8 +297,8 @@ fn main() {
         .allowlist_function("rb_yjit_mark_executable")
         .allowlist_function("rb_yjit_mark_unused")
         .allowlist_function("rb_yjit_get_page_size")
-        .allowlist_function("rb_leaf_invokebuiltin_iseq_p")
-        .allowlist_function("rb_leaf_builtin_function")
+        .allowlist_function("rb_yjit_iseq_builtin_attrs")
+        .allowlist_function("rb_yjit_builtin_function")
         .allowlist_function("rb_set_cfp_(pc|sp)")
         .allowlist_function("rb_cfp_get_iseq")
         .allowlist_function("rb_yjit_multi_ractor_p")

yjit/src/codegen.rs

@@ -5557,14 +5557,10 @@ fn gen_send_iseq(
         }
     }
 
-    let leaf_builtin_raw = unsafe { rb_leaf_builtin_function(iseq) };
-    let leaf_builtin: Option<*const rb_builtin_function> = if leaf_builtin_raw.is_null() {
-        None
-    } else {
-        Some(leaf_builtin_raw)
-    };
-    if let (None, Some(builtin_info)) = (block, leaf_builtin) {
-
+    let builtin_attrs = unsafe { rb_yjit_iseq_builtin_attrs(iseq) };
+    let builtin_func_raw = unsafe { rb_yjit_builtin_function(iseq) };
+    let builtin_func = if builtin_func_raw.is_null() { None } else { Some(builtin_func_raw) };
+    if let (None, Some(builtin_info), true) = (block, builtin_func, builtin_attrs & BUILTIN_ATTR_LEAF != 0) {
         // this is a .send call not currently supported for builtins
         if flags & VM_CALL_OPT_SEND != 0 {
             gen_counter_incr!(asm, send_send_builtin);
@@ -5575,9 +5571,13 @@ fn gen_send_iseq(
         if builtin_argc + 1 < (C_ARG_OPNDS.len() as i32) {
             asm.comment("inlined leaf builtin");
 
-            // Save the PC and SP because the callee may allocate
-            // e.g. Integer#abs on a bignum
-            jit_prepare_routine_call(jit, ctx, asm);
+            // Skip gen_save_sp if it doesn't trigger GC
+            if builtin_attrs & BUILTIN_ATTR_NO_GC == 0 {
+                // The callee may allocate, e.g. Integer#abs on a Bignum.
+                // Save SP for GC, save PC for allocation tracing, and prepare
+                // for global invalidation after GC's VM lock contention.
+                jit_prepare_routine_call(jit, ctx, asm);
+            }
 
             // Call the builtin func (ec, recv, arg1, arg2, ...)
             let mut args = vec![EC];

yjit/src/cruby_bindings.inc.rs

@@ -662,6 +662,9 @@ pub struct iseq_inline_iv_cache_entry {
 pub struct iseq_inline_cvar_cache_entry {
     pub entry: *mut rb_cvar_class_tbl_entry,
 }
+pub const BUILTIN_ATTR_LEAF: rb_builtin_attr = 1;
+pub const BUILTIN_ATTR_NO_GC: rb_builtin_attr = 2;
+pub type rb_builtin_attr = u32;
 #[repr(C)]
 #[derive(Debug, Copy, Clone)]
 pub struct rb_iseq_constant_body__bindgen_ty_1_rb_iseq_param_keyword {
@@ -1276,8 +1279,8 @@ extern "C" {
         kw_splat: ::std::os::raw::c_int,
         block_handler: VALUE,
     ) -> VALUE;
-    pub fn rb_leaf_invokebuiltin_iseq_p(iseq: *const rb_iseq_t) -> bool;
-    pub fn rb_leaf_builtin_function(iseq: *const rb_iseq_t) -> *const rb_builtin_function;
+    pub fn rb_yjit_iseq_builtin_attrs(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint;
+    pub fn rb_yjit_builtin_function(iseq: *const rb_iseq_t) -> *const rb_builtin_function;
     pub fn rb_yjit_str_simple_append(str1: VALUE, str2: VALUE) -> VALUE;
     pub fn rb_get_ec_cfp(ec: *const rb_execution_context_t) -> *mut rb_control_frame_struct;
     pub fn rb_get_cfp_pc(cfp: *mut rb_control_frame_struct) -> *mut VALUE;