YJIT: Introduce no_gc attribute (#7511)

compile.c

@@ -8236,6 +8236,9 @@ compile_builtin_attr(rb_iseq_t *iseq, const NODE *node)
         if (strcmp(RSTRING_PTR(string), "leaf") == 0) {
             ISEQ_BODY(iseq)->builtin_attrs |= BUILTIN_ATTR_LEAF;
         }
+        else if (strcmp(RSTRING_PTR(string), "no_gc") == 0) {
+            ISEQ_BODY(iseq)->builtin_attrs |= BUILTIN_ATTR_NO_GC;
+        }
         else {
             goto unknown_arg;
         }

kernel.rb

@@ -16,7 +16,7 @@ module Kernel
   #++
   #
   def class
-    Primitive.attr! :leaf
+    Primitive.attr! :leaf, :no_gc
     Primitive.cexpr! 'rb_obj_class(self)'
   end
 

tool/mk_builtin_loader.rb

@@ -6,7 +6,7 @@
 
 SUBLIBS = {}
 REQUIRED = {}
-BUILTIN_ATTRS = %w[leaf]
+BUILTIN_ATTRS = %w[leaf no_gc]
 
 def string_literal(lit, str = [])
   while lit

vm_core.h

@@ -370,8 +370,10 @@ enum rb_iseq_type {
 
 // Attributes specified by Primitive.attr!
 enum rb_builtin_attr {
-    // If true, this ISeq does not call methods.
+    // The iseq does not call methods.
     BUILTIN_ATTR_LEAF = 0x01,
+    // The iseq does not allocate objects.
+    BUILTIN_ATTR_NO_GC = 0x02,
 };
 
 struct rb_iseq_constant_body {

yjit.c

@@ -724,28 +724,33 @@ rb_optimized_call(VALUE *recv, rb_execution_context_t *ec, int argc, VALUE *argv
     return rb_vm_invoke_proc(ec, proc, argc, argv, kw_splat, block_handler);
 }
 
+unsigned int
+rb_yjit_iseq_builtin_attrs(const rb_iseq_t *iseq)
+{
+    return iseq->body->builtin_attrs;
+}
 
-// If true, the iseq is leaf and it can be replaced by a single C call.
-bool
-rb_leaf_invokebuiltin_iseq_p(const rb_iseq_t *iseq)
+// If true, the iseq has only opt_invokebuiltin_delegate_leave and leave insns.
+static bool
+invokebuiltin_delegate_leave_p(const rb_iseq_t *iseq)
 {
     unsigned int invokebuiltin_len = insn_len(BIN(opt_invokebuiltin_delegate_leave));
     unsigned int leave_len = insn_len(BIN(leave));
-
-    return (iseq->body->iseq_size == (invokebuiltin_len + leave_len) &&
+    return iseq->body->iseq_size == (invokebuiltin_len + leave_len) &&
         rb_vm_insn_addr2opcode((void *)iseq->body->iseq_encoded[0]) == BIN(opt_invokebuiltin_delegate_leave) &&
-        rb_vm_insn_addr2opcode((void *)iseq->body->iseq_encoded[invokebuiltin_len]) == BIN(leave) &&
-        (iseq->body->builtin_attrs & BUILTIN_ATTR_LEAF) != 0
-    );
+        rb_vm_insn_addr2opcode((void *)iseq->body->iseq_encoded[invokebuiltin_len]) == BIN(leave);
 }
 
-// Return an rb_builtin_function if the iseq contains only that leaf builtin function.
+// Return an rb_builtin_function if the iseq contains only that builtin function.
 const struct rb_builtin_function *
-rb_leaf_builtin_function(const rb_iseq_t *iseq)
+rb_yjit_builtin_function(const rb_iseq_t *iseq)
 {
-    if (!rb_leaf_invokebuiltin_iseq_p(iseq))
+    if (invokebuiltin_delegate_leave_p(iseq)) {
+        return (const struct rb_builtin_function *)iseq->body->iseq_encoded[1];
+    }
+    else {
         return NULL;
-    return (const struct rb_builtin_function *)iseq->body->iseq_encoded[1];
+    }
 }
 
 VALUE

yjit/bindgen/src/main.rs

@@ -286,6 +286,7 @@ fn main() {
         .allowlist_var("VM_ENV_DATA_INDEX_FLAGS")
         .allowlist_var("VM_ENV_DATA_SIZE")
         .allowlist_function("rb_iseq_path")
+        .allowlist_type("rb_builtin_attr")
 
         // From yjit.c
         .allowlist_function("rb_iseq_(get|set)_yjit_payload")
@@ -296,8 +297,8 @@ fn main() {
         .allowlist_function("rb_yjit_mark_executable")
         .allowlist_function("rb_yjit_mark_unused")
         .allowlist_function("rb_yjit_get_page_size")
-        .allowlist_function("rb_leaf_invokebuiltin_iseq_p")
-        .allowlist_function("rb_leaf_builtin_function")
+        .allowlist_function("rb_yjit_iseq_builtin_attrs")
+        .allowlist_function("rb_yjit_builtin_function")
         .allowlist_function("rb_set_cfp_(pc|sp)")
         .allowlist_function("rb_cfp_get_iseq")
         .allowlist_function("rb_yjit_multi_ractor_p")

yjit/src/codegen.rs

@@ -5591,14 +5591,10 @@ fn gen_send_iseq(
         }
     }
 
-    let leaf_builtin_raw = unsafe { rb_leaf_builtin_function(iseq) };
-    let leaf_builtin: Option<*const rb_builtin_function> = if leaf_builtin_raw.is_null() {
-        None
-    } else {
-        Some(leaf_builtin_raw)
-    };
-    if let (None, Some(builtin_info)) = (block, leaf_builtin) {
-
+    let builtin_attrs = unsafe { rb_yjit_iseq_builtin_attrs(iseq) };
+    let builtin_func_raw = unsafe { rb_yjit_builtin_function(iseq) };
+    let builtin_func = if builtin_func_raw.is_null() { None } else { Some(builtin_func_raw) };
+    if let (None, Some(builtin_info), true) = (block, builtin_func, builtin_attrs & BUILTIN_ATTR_LEAF != 0) {
         // this is a .send call not currently supported for builtins
         if flags & VM_CALL_OPT_SEND != 0 {
             gen_counter_incr!(asm, send_send_builtin);
@@ -5609,9 +5605,13 @@ fn gen_send_iseq(
         if builtin_argc + 1 < (C_ARG_OPNDS.len() as i32) {
             asm.comment("inlined leaf builtin");
 
-            // Save the PC and SP because the callee may allocate
-            // e.g. Integer#abs on a bignum
-            jit_prepare_routine_call(jit, ctx, asm);
+            // Skip this if it doesn't trigger GC
+            if builtin_attrs & BUILTIN_ATTR_NO_GC == 0 {
+                // The callee may allocate, e.g. Integer#abs on a Bignum.
+                // Save SP for GC, save PC for allocation tracing, and prepare
+                // for global invalidation after GC's VM lock contention.
+                jit_prepare_routine_call(jit, ctx, asm);
+            }
 
             // Call the builtin func (ec, recv, arg1, arg2, ...)
             let mut args = vec![EC];

yjit/src/cruby_bindings.inc.rs

@@ -662,6 +662,9 @@ pub struct iseq_inline_iv_cache_entry {
 pub struct iseq_inline_cvar_cache_entry {
     pub entry: *mut rb_cvar_class_tbl_entry,
 }
+pub const BUILTIN_ATTR_LEAF: rb_builtin_attr = 1;
+pub const BUILTIN_ATTR_NO_GC: rb_builtin_attr = 2;
+pub type rb_builtin_attr = u32;
 #[repr(C)]
 #[derive(Debug, Copy, Clone)]
 pub struct rb_iseq_constant_body__bindgen_ty_1_rb_iseq_param_keyword {
@@ -1276,8 +1279,8 @@ extern "C" {
         kw_splat: ::std::os::raw::c_int,
         block_handler: VALUE,
     ) -> VALUE;
-    pub fn rb_leaf_invokebuiltin_iseq_p(iseq: *const rb_iseq_t) -> bool;
-    pub fn rb_leaf_builtin_function(iseq: *const rb_iseq_t) -> *const rb_builtin_function;
+    pub fn rb_yjit_iseq_builtin_attrs(iseq: *const rb_iseq_t) -> ::std::os::raw::c_uint;
+    pub fn rb_yjit_builtin_function(iseq: *const rb_iseq_t) -> *const rb_builtin_function;
     pub fn rb_yjit_str_simple_append(str1: VALUE, str2: VALUE) -> VALUE;
     pub fn rb_get_ec_cfp(ec: *const rb_execution_context_t) -> *mut rb_control_frame_struct;
     pub fn rb_get_cfp_pc(cfp: *mut rb_control_frame_struct) -> *mut VALUE;