[pull] master from ruby:master

encoding.c

@@ -98,6 +98,7 @@ static rb_encoding *global_enc_ascii,
                    *global_enc_us_ascii;
 
 static int filesystem_encindex = ENCINDEX_ASCII_8BIT;
+static rb_atomic_t locale_alias_registered;
 
 #define GLOBAL_ENC_TABLE_LOCKING(tbl) \
     for (struct enc_table *tbl = &global_enc_table, **locking = &tbl; \
@@ -1568,15 +1569,16 @@ rb_locale_encindex(void)
 
     if (idx < 0) idx = ENCINDEX_UTF_8;
 
-    GLOBAL_ENC_TABLE_LOCKING(enc_table) {
-        if (enc_registered(enc_table, "locale") < 0) {
+    if (!RUBY_ATOMIC_LOAD(locale_alias_registered)) {
+        GLOBAL_ENC_TABLE_LOCKING(enc_table) {
+            if (enc_registered(enc_table, "locale") < 0) {
 # if defined _WIN32
-            void Init_w32_codepage(void);
-            Init_w32_codepage();
+                void Init_w32_codepage(void);
+                Init_w32_codepage();
 # endif
-            GLOBAL_ENC_TABLE_LOCKING(enc_table) {
                 enc_alias_internal(enc_table, "locale", idx);
             }
+            RUBY_ATOMIC_SET(locale_alias_registered, 1);
         }
     }
 

ext/openssl/ossl_kdf.c

@@ -39,6 +39,10 @@ pbkdf2_hmac_nogvl(void *args_)
  * For more information about PBKDF2, see RFC 2898 Section 5.2
  * (https://www.rfc-editor.org/rfc/rfc2898#section-5.2).
  *
+ * *NOTE*: This method cannot be interrupted by Timeout.timeout from the
+ * "timeout" library. Do not take parameters from untrusted sources without
+ * enforcing reasonable limits.
+ *
  * === Parameters
  * pass       :: The password.
  * salt       :: The salt. Salts prevent attacks based on dictionaries of common
@@ -143,6 +147,10 @@ scrypt_nogvl(void *args_)
  *
  * See RFC 7914 (https://www.rfc-editor.org/rfc/rfc7914) for more information.
  *
+ * *NOTE*: This method cannot be interrupted by Timeout.timeout from the
+ * "timeout" library. Do not take parameters from untrusted sources without
+ * enforcing reasonable limits.
+ *
  * === Parameters
  * pass   :: Passphrase.
  * salt   :: Salt.

include/ruby/internal/abi.h

@@ -24,7 +24,7 @@
  * In released versions of Ruby, this number is not defined since teeny
  * versions of Ruby should guarantee ABI compatibility.
  */
-#define RUBY_ABI_VERSION 2
+#define RUBY_ABI_VERSION 3
 
 /* Windows does not support weak symbols so ruby_abi_version will not exist
  * in the shared library. */

vm.c

@@ -574,13 +574,6 @@ jit_exec(rb_execution_context_t *ec)
         rb_jit_func_t func = zjit_compile(ec);
         if (func) {
             VALUE result = ((rb_zjit_func_t)zjit_entry)(ec, ec->cfp, func);
-            // Materialize any remaining lightweight ZJIT frames on side exit.
-            // This is done here (once per JIT entry) instead of in each side exit
-            // to reduce generated code size.
-            if (UNDEF_P(result)) {
-                ec->cfp->jit_return = 0; // exit code already cleared most fields except jit_return
-                rb_zjit_materialize_frames(ec, ec->cfp);
-            }
             return result;
         }
     }

vm_exec.h

@@ -189,10 +189,6 @@ default:                        \
             rb_jit_func_t func = zjit_compile(ec); \
             if (func) { \
                 val = zjit_entry(ec, ec->cfp, func); \
-                if (UNDEF_P(val)) { \
-                    ec->cfp->jit_return = 0; \
-                    rb_zjit_materialize_frames(ec, ec->cfp); \
-                } \
             } \
         } \
     } \

zjit/src/backend/lir.rs

@@ -4,7 +4,7 @@ use std::mem::take;
 use std::rc::Rc;
 use crate::bitset::BitSet;
 use crate::codegen::{local_size_and_idx_to_ep_offset, perf_symbol_range_start, perf_symbol_range_end};
-use crate::cruby::{IseqPtr, Qundef, RUBY_OFFSET_CFP_ISEQ, RUBY_OFFSET_CFP_JIT_RETURN, RUBY_OFFSET_CFP_PC, RUBY_OFFSET_CFP_SP, SIZEOF_VALUE_I32, vm_stack_canary};
+use crate::cruby::{IseqPtr, RUBY_OFFSET_CFP_ISEQ, RUBY_OFFSET_CFP_JIT_RETURN, RUBY_OFFSET_CFP_PC, RUBY_OFFSET_CFP_SP, SIZEOF_VALUE_I32, vm_stack_canary};
 use crate::hir::{Invariant, SideExitReason};
 use crate::hir;
 use crate::options::{TraceExits, PerfMap, get_option};
@@ -13,7 +13,7 @@ use crate::payload::IseqVersionRef;
 use crate::stats::{exit_counter_ptr, exit_counter_ptr_for_opcode, side_exit_counter, CompileError};
 use crate::virtualmem::CodePtr;
 use crate::asm::{CodeBlock, Label};
-use crate::state::rb_zjit_record_exit_stack;
+use crate::state::{ZJITState, rb_zjit_record_exit_stack};
 
 /// LIR Block ID. Unique ID for each block, and also defined in LIR so
 /// we can differentiate it from HIR block ids.
@@ -1562,23 +1562,6 @@ impl Assembler
         iter
     }
 
-    /// Return an operand for a basic block argument at a given index.
-    /// To simplify the implementation, we allocate a fixed register or a stack slot
-    /// for each basic block argument.
-    pub fn param_opnd(idx: usize) -> Opnd {
-        use crate::backend::current::ALLOC_REGS;
-        use crate::cruby::SIZEOF_VALUE_I32;
-
-        if idx < ALLOC_REGS.len() {
-            Opnd::Reg(ALLOC_REGS[idx])
-        } else {
-            // With FrameSetup, the address that NATIVE_BASE_PTR points to stores an old value in the register.
-            // To avoid clobbering it, we need to start from the next slot, and we also reserve one space for
-            // JITFrame, hence `+ 2` for the index.
-            Opnd::mem(64, NATIVE_BASE_PTR, (idx - ALLOC_REGS.len() + 2) as i32 * -SIZEOF_VALUE_I32)
-        }
-    }
-
     pub fn linearize_instructions(&self) -> Vec<Insn> {
         // Emit instructions with labels, expanding branch parameters
         let mut insns = Vec::with_capacity(ASSEMBLER_INSNS_CAPACITY);
@@ -2040,11 +2023,25 @@ impl Assembler
             if self.basic_blocks[block_id.0].is_dummy() { continue; }
             let params = self.basic_blocks[block_id.0].parameters.clone();
 
+            // JIT-to-JIT entries that would need more argument registers should
+            // be unreachable because can_direct_send() refuses to call them.
+            // Keep compiling the function body, but make the unsupported entry
+            // abort if control ever reaches it. TODO: Remove this (Shopify/ruby#916)
+            if params.len() > C_ARG_OPNDS.len() {
+                let insert_pos = self.basic_blocks[block_id.0].insns.iter()
+                    .position(|insn| matches!(insn, Insn::FrameSetup { .. }))
+                    .or_else(|| self.basic_blocks[block_id.0].insns.iter().position(|insn| matches!(insn, Insn::Label(_))).map(|idx| idx + 1))
+                    .unwrap_or(0);
+                self.basic_blocks[block_id.0].insns.insert(insert_pos, Insn::Abort);
+                self.basic_blocks[block_id.0].insn_ids.insert(insert_pos, None);
+                continue;
+            }
+
             // Rewrite VRegs to physical registers before sequentialization
             // so the parcopy algorithm can detect physical register conflicts.
             let reg_copies: Vec<parcopy::RegisterCopy<Opnd>> = params.iter().enumerate()
                 .map(|(i, param)| parcopy::RegisterCopy::<Opnd> {
-                    source: Assembler::param_opnd(i),
+                    source: C_ARG_OPNDS[i],
                     destination: Self::rewritten_opnd(*param, assignments),
                 })
                 .filter(|copy| copy.source != copy.destination)
@@ -2380,7 +2377,7 @@ impl Assembler
             asm_comment!(asm, "save cfp->iseq");
             asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_ISEQ), VALUE::from(*iseq).into());
 
-            // cfp->block_code and cfp->jit_return are cleared by the caller jit_exec() or JIT_EXEC()
+            // cfp->block_code and cfp->jit_return are cleared by the materialize_exit trampoline
 
             if !stack.is_empty() {
                 asm_comment!(asm, "write stack slots: {}", join_opnds(&stack, ", "));
@@ -2400,8 +2397,7 @@ impl Assembler
         /// Tear down the JIT frame and return to the interpreter.
         fn compile_exit_return(asm: &mut Assembler) {
             asm_comment!(asm, "exit to the interpreter");
-            asm.frame_teardown(&[]); // matching the setup in gen_entry_point()
-            asm.cret(Opnd::UImm(Qundef.as_u64()));
+            asm.jmp(Target::CodePtr(ZJITState::get_materialize_exit_trampoline()));
         }
 
         fn compile_exit_recompile(asm: &mut Assembler, exit: &SideExit) {
@@ -2434,7 +2430,7 @@ impl Assembler
             compile_exit_save_state(asm, exit);
             if trace_reason.is_some() || exit.recompile.is_some() {
                 // Clear cfp->jit_return to prepare for a C call. Normally, cfp->jit_return
-                // is cleared by the caller jit_exec() or JIT_EXEC(), but if we're about to
+                // is cleared by the materialize_exit trampoline, but if we're about to
                 // make a C call, we need to clear any stale JITFrame.
                 asm_comment!(asm, "clear cfp->jit_return");
                 asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), 0.into());
@@ -4217,8 +4213,8 @@ mod tests {
         let (assignments, _) = asm.linear_scan(intervals.clone(), 5, &preferred_registers);
 
         // Entry block b1 has parameters [v0, v1].
-        // With 5 registers: v0 -> Reg(0) = regs[0], arrival = param_opnd(0) = regs[0] -> self-move, filtered
-        //                    v1 -> Reg(1) = regs[1], arrival = param_opnd(1) = regs[1] -> self-move, filtered
+        // With 5 registers: v0 -> Reg(0) = regs[0], arrival = C_ARG_OPNDS[0] = regs[0] -> self-move, filtered
+        //                    v1 -> Reg(1) = regs[1], arrival = C_ARG_OPNDS[1] = regs[1] -> self-move, filtered
         // Before resolve_ssa, b1 has: [Label, Jmp] = 2 insns
         assert_eq!(asm.basic_blocks[b1.0].insns.len(), 2);
 
@@ -4243,6 +4239,31 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_resolve_ssa_entry_params_too_many_abort() {
+        let mut asm = Assembler::new();
+        let block = asm.new_block(hir::BlockId(0), true, 0);
+        asm.set_current_block(block);
+        let label = asm.new_label("bb0");
+        asm.write_label(label);
+
+        for _ in 0..=C_ARG_OPNDS.len() {
+            let param = asm.new_vreg(64);
+            asm.basic_blocks[block.0].add_parameter(param);
+        }
+        asm.basic_blocks[block.0].push_insn(Insn::CRet(Opnd::UImm(0)));
+
+        let live_in = asm.analyze_liveness();
+        asm.number_instructions(0);
+        let intervals = asm.build_intervals(live_in);
+        let preferred_registers = asm.preferred_register_assignments(&intervals);
+        let (assignments, _) = asm.linear_scan(intervals.clone(), 5, &preferred_registers);
+
+        asm.resolve_ssa(&intervals, &assignments);
+
+        assert!(matches!(asm.basic_blocks[block.0].insns[1], Insn::Abort));
+    }
+
     fn build_critical_edge() -> (Assembler, Opnd, Opnd, Opnd, Opnd, Opnd, BlockId, BlockId, BlockId) {
         let mut asm = Assembler::new();
 

zjit/src/codegen.rs

@@ -2551,25 +2551,6 @@ fn gen_guard_type(jit: &mut JITState, asm: &mut Assembler, val: lir::Opnd, guard
 
         asm.cmp(klass, Opnd::Value(expected_class));
         asm.jne(jit, side_exit);
-    } else if guard_type.is_subtype(types::TData) {
-        let side = side_exit(jit, state, GuardType(guard_type));
-
-        // Check special constant
-        asm.test(val, Opnd::UImm(RUBY_IMMEDIATE_MASK as u64));
-        asm.jnz(jit, side.clone());
-
-        // Check false
-        asm.cmp(val, Qfalse.into());
-        asm.je(jit, side.clone());
-
-        // Check the T_DATA builtin type.
-        let val = asm.load_mem(val);
-        let flags = asm.load(Opnd::mem(VALUE_BITS, val, RUBY_OFFSET_RBASIC_FLAGS));
-        let mask     = RUBY_T_MASK.to_usize();
-        let expected = RUBY_T_DATA.to_usize();
-        let masked = asm.and(flags, mask.into());
-        asm.cmp(masked, expected.into());
-        asm.jne(jit, side);
     } else if let Some(builtin_type) = guard_type.builtin_type_equivalent() {
         let side = side_exit(jit, state, GuardType(guard_type));
 
@@ -3098,7 +3079,7 @@ c_callable! {
             // If we side-exit from function_stub_hit (before JIT code runs), we need to set them here.
             fn prepare_for_exit(iseq: IseqPtr, cfp: CfpPtr, sp: *mut VALUE, argc: u16, num_opts_filled: u16, compile_error: &CompileError) {
                 unsafe {
-                    // Caller frames are materialized by jit_exec() after the entry trampoline returns.
+                    // Caller frames are materialized by the materialize_exit trampoline before unwinding native frames.
                     // The current frame's pc and iseq are already set by function_stub_hit before this point.
 
                     // Set SP which gen_push_frame() doesn't set
@@ -3186,7 +3167,7 @@ c_callable! {
                 unsafe { Rc::increment_strong_count(iseq_call_ptr as *const IseqCall); }
 
                 prepare_for_exit(iseq, cfp, sp, argc, num_opts_filled, compile_error);
-                return ZJITState::get_exit_trampoline_with_counter().raw_ptr(cb);
+                return ZJITState::get_materialize_exit_trampoline_with_counter().raw_ptr(cb);
             }
 
             // Otherwise, attempt to compile the ISEQ. We have to mark_all_executable() beyond this point.
@@ -3201,7 +3182,7 @@ c_callable! {
                 unsafe { Rc::increment_strong_count(iseq_call_ptr as *const IseqCall); }
 
                 prepare_for_exit(iseq, cfp, sp, argc, num_opts_filled, &compile_error);
-                ZJITState::get_exit_trampoline_with_counter()
+                ZJITState::get_materialize_exit_trampoline_with_counter()
             });
             cb.mark_all_executable();
             code_ptr.raw_ptr(cb)
@@ -3332,14 +3313,34 @@ pub fn gen_exit_trampoline(cb: &mut CodeBlock) -> Result<CodePtr, CompileError>
     })
 }
 
-/// Generate a trampoline that increments exit_compilation_failure and jumps to exit_trampoline.
-pub fn gen_exit_trampoline_with_counter(cb: &mut CodeBlock, exit_trampoline: CodePtr) -> Result<CodePtr, CompileError> {
+/// Generate a trampoline that materializes ZJIT frames before unwinding native frames.
+pub fn gen_materialize_exit_trampoline(cb: &mut CodeBlock, exit_trampoline: CodePtr) -> Result<CodePtr, CompileError> {
+    unsafe extern "C" {
+        fn rb_zjit_materialize_frames(ec: EcPtr, cfp: CfpPtr);
+    }
+
     let mut asm = Assembler::new();
-    asm.new_block_without_id("exit_trampoline_with_counter");
+    asm.new_block_without_id("materialize_exit_trampoline");
+
+    asm_comment!(asm, "materialize ZJIT frames");
+    asm.store(Opnd::mem(64, CFP, RUBY_OFFSET_CFP_JIT_RETURN), 0.into());
+    asm_ccall!(asm, rb_zjit_materialize_frames, EC, CFP);
+    asm.jmp(Target::CodePtr(exit_trampoline));
+
+    asm.compile(cb).map(|(code_ptr, gc_offsets)| {
+        assert_eq!(gc_offsets.len(), 0);
+        code_ptr
+    })
+}
+
+/// Generate a trampoline that increments exit_compilation_failure and jumps to materialize_exit_trampoline.
+pub fn gen_materialize_exit_trampoline_with_counter(cb: &mut CodeBlock, materialize_exit_trampoline: CodePtr) -> Result<CodePtr, CompileError> {
+    let mut asm = Assembler::new();
+    asm.new_block_without_id("materialize_exit_trampoline_with_counter");
 
     asm_comment!(asm, "function stub exit trampoline");
     gen_incr_counter(&mut asm, exit_compile_error);
-    asm.jmp(Target::CodePtr(exit_trampoline));
+    asm.jmp(Target::CodePtr(materialize_exit_trampoline));
 
     asm.compile(cb).map(|(code_ptr, gc_offsets)| {
         assert_eq!(gc_offsets.len(), 0);

zjit/src/hir_type/mod.rs

@@ -444,8 +444,9 @@ impl Type {
             Some(cruby::RUBY_T_STRING)
         } else if self.bit_equal(types::Hash) {
             Some(cruby::RUBY_T_HASH)
+        } else if self.bit_equal(types::TData) {
+            Some(cruby::RUBY_T_DATA)
         } else {
-            // T_DATA uses a specialized guard, so not here.
             None
         }
     }

zjit/src/state.rs

@@ -1,6 +1,6 @@
 //! Runtime state of ZJIT.
 
-use crate::codegen::{gen_entry_trampoline, gen_exit_trampoline, gen_exit_trampoline_with_counter, gen_function_stub_hit_trampoline};
+use crate::codegen::{gen_entry_trampoline, gen_exit_trampoline, gen_function_stub_hit_trampoline, gen_materialize_exit_trampoline, gen_materialize_exit_trampoline_with_counter};
 use crate::cruby::{self, rb_bug_panic_hook, rb_vm_insn_count, src_loc, EcPtr, Qnil, Qtrue, rb_profile_frames, rb_profile_frame_full_label, rb_profile_frame_absolute_path, rb_profile_frame_path, VALUE, VM_INSTRUCTION_SIZE, with_vm_lock, rust_str_to_id, rb_funcallv, rb_const_get, rb_cRubyVM};
 use crate::cruby_methods;
 use cruby::{ID, rb_callable_method_entry, get_def_method_serial, rb_gc_register_mark_object, ruby_str_to_rust_string_result};
@@ -51,8 +51,11 @@ pub struct ZJITState {
     /// Trampoline to side-exit without restoring PC or the stack
     exit_trampoline: CodePtr,
 
-    /// Trampoline to side-exit and increment exit_compilation_failure
-    exit_trampoline_with_counter: CodePtr,
+    /// Trampoline to materialize JIT frames before side-exiting
+    materialize_exit_trampoline: CodePtr,
+
+    /// Trampoline to materialize JIT frames and increment exit_compilation_failure
+    materialize_exit_trampoline_with_counter: CodePtr,
 
     /// Trampoline to call function_stub_hit
     function_stub_hit_trampoline: CodePtr,
@@ -126,6 +129,7 @@ impl ZJITState {
 
         let entry_trampoline = gen_entry_trampoline(&mut cb).unwrap().raw_ptr(&cb);
         let exit_trampoline = gen_exit_trampoline(&mut cb).unwrap();
+        let materialize_exit_trampoline = gen_materialize_exit_trampoline(&mut cb, exit_trampoline).unwrap();
         let function_stub_hit_trampoline = gen_function_stub_hit_trampoline(&mut cb).unwrap();
 
         let perfetto_tracer = if get_option!(trace_side_exits).is_some() || get_option!(trace_compiles) || get_option!(trace_invalidation) {
@@ -144,8 +148,9 @@ impl ZJITState {
             assert_compiles: false,
             method_annotations,
             exit_trampoline,
+            materialize_exit_trampoline,
+            materialize_exit_trampoline_with_counter: materialize_exit_trampoline,
             function_stub_hit_trampoline,
-            exit_trampoline_with_counter: exit_trampoline,
             full_frame_cfunc_counter_pointers: HashMap::new(),
             not_annotated_frame_cfunc_counter_pointers: HashMap::new(),
             ccall_counter_pointers: HashMap::new(),
@@ -160,8 +165,8 @@ impl ZJITState {
         // on the counter, so ZJIT_STATE needs to be initialized first.
         if get_option!(stats) {
             let cb = ZJITState::get_code_block();
-            let code_ptr = gen_exit_trampoline_with_counter(cb, exit_trampoline).unwrap();
-            ZJITState::get_instance().exit_trampoline_with_counter = code_ptr;
+            let code_ptr = gen_materialize_exit_trampoline_with_counter(cb, materialize_exit_trampoline).unwrap();
+            ZJITState::get_instance().materialize_exit_trampoline_with_counter = code_ptr;
         }
 
         entry_trampoline
@@ -288,9 +293,14 @@ impl ZJITState {
         ZJITState::get_instance().exit_trampoline
     }
 
-    /// Return a code pointer to the exit trampoline for function stubs
-    pub fn get_exit_trampoline_with_counter() -> CodePtr {
-        ZJITState::get_instance().exit_trampoline_with_counter
+    /// Return a code pointer to the materialize_exit trampoline
+    pub fn get_materialize_exit_trampoline() -> CodePtr {
+        ZJITState::get_instance().materialize_exit_trampoline
+    }
+
+    /// Return a code pointer to the materialize_exit trampoline for function stubs
+    pub fn get_materialize_exit_trampoline_with_counter() -> CodePtr {
+        ZJITState::get_instance().materialize_exit_trampoline_with_counter
     }
 
     /// Return a code pointer to the function stub hit trampoline