[pull] master from ruby:master

NEWS.md

@@ -88,6 +88,7 @@ releases.
   * 4.0.0 to [v4.0.1][openssl-v4.0.1], [v4.0.2][openssl-v4.0.2]
 * prism 1.9.0
   * 1.7.0 to [v1.8.0][prism-v1.8.0], [v1.8.1][prism-v1.8.1], [v1.9.0][prism-v1.9.0]
+* psych 5.4.0
 * resolv 0.7.1
   * 0.7.0 to [v0.7.1][resolv-v0.7.1]
 * stringio 3.2.1.dev

array.c

@@ -30,6 +30,7 @@
 #include "ruby/thread.h"
 #include "ruby/util.h"
 #include "ruby/ractor.h"
+#include "shape.h"
 #include "vm_core.h"
 #include "builtin.h"
 
@@ -909,6 +910,7 @@ init_fake_ary_flags(void)
     struct RArray fake_ary = {0};
     fake_ary.basic.flags = T_ARRAY;
     VALUE ary = (VALUE)&fake_ary;
+    RBASIC_SET_SHAPE_ID(ary, ROOT_SHAPE_ID | SHAPE_ID_LAYOUT_OTHER);
     rb_ary_freeze(ary);
     return fake_ary.basic.flags;
 }
@@ -2682,6 +2684,12 @@ ary_enum_length(VALUE ary, VALUE args, VALUE eobj)
     return rb_ary_length(ary);
 }
 
+// These array primitives enable tight compatibility with the C implementation
+// in terms of what method calls happen. They can use unchecked utilities such as
+// FIX2LONG since unlike userland Ruby code, these methods cannot be traced with
+// TracePoint (or ruby/debug.h APIs) and have their local variables changed from
+// underneath them.
+
 // Return true if the index is at or past the end of the array.
 VALUE
 rb_jit_ary_at_end(rb_execution_context_t *ec, VALUE self, VALUE index)

benchmark/string_coderange_scan.yml

@@ -0,0 +1,10 @@
+prelude: |
+  def unknown(s) = s.b.force_encoding("UTF-8")
+  multibyte   = unknown("\u{00e9}" * 16384)   # best case: every byte non-ASCII
+  alternating = unknown("\u{00e9}a" * 10922)  # worst case: non-ASCII then ASCII
+  ascii       = unknown("a" * 32768)          # baseline
+
+benchmark:
+  coderange_multibyte:   multibyte.dup.valid_encoding?
+  coderange_alternating: alternating.dup.valid_encoding?
+  coderange_ascii:       ascii.dup.valid_encoding?

class.c

@@ -585,7 +585,15 @@ class_alloc0(enum ruby_value_type type, VALUE klass, bool boxable)
     VALUE flags = type | FL_SHAREABLE;
     if (boxable) flags |= RCLASS_BOXABLE;
 
-    NEWOBJ_OF(obj, struct RClass, klass, flags, alloc_size);
+    shape_id_t shape_id = ROOT_SHAPE_ID;
+    if (boxable) {
+        shape_id |= SHAPE_ID_LAYOUT_OTHER;
+    }
+    else {
+        shape_id |= SHAPE_ID_LAYOUT_RCLASS;
+    }
+
+    struct RClass *obj = (struct RClass *)rb_newobj(GET_EC(), klass, flags, shape_id, true, alloc_size);
 
     obj->object_id = 0;
 

depend

@@ -80,6 +80,7 @@ array.$(OBJEXT): $(top_srcdir)/internal/sanitizers.h
 array.$(OBJEXT): $(top_srcdir)/internal/serial.h
 array.$(OBJEXT): $(top_srcdir)/internal/set_table.h
 array.$(OBJEXT): $(top_srcdir)/internal/static_assert.h
+array.$(OBJEXT): $(top_srcdir)/internal/struct.h
 array.$(OBJEXT): $(top_srcdir)/internal/variable.h
 array.$(OBJEXT): $(top_srcdir)/internal/vm.h
 array.$(OBJEXT): $(top_srcdir)/internal/warnings.h

ext/psych/lib/psych/versions.rb

@@ -2,7 +2,7 @@
 
 module Psych
   # The version of Psych you are using
-  VERSION = '5.3.1'
+  VERSION = '5.4.0'
 
   if RUBY_ENGINE == 'jruby'
     DEFAULT_SNAKEYAML_VERSION = '2.10'.freeze

ext/psych/psych_parser.c

@@ -32,9 +32,20 @@ static int io_reader(void * data, unsigned char *buf, size_t size, size_t *read)
     *read = 0;
 
     if(! NIL_P(string)) {
-        void * str = (void *)StringValuePtr(string);
-        *read = (size_t)RSTRING_LEN(string);
-        memcpy(buf, str, *read);
+        char * str = StringValuePtr(string);
+        size_t len = (size_t)RSTRING_LEN(string);
+
+        /* IO#read(size) is documented to return at most `size` bytes, but a
+         * misbehaving IO-like object may return more. Clamp the copy to the
+         * buffer libyaml gave us to avoid writing past its end, rounding down
+         * to a character boundary so a multibyte character is never split. */
+        if(len > size) {
+            rb_encoding * enc = rb_enc_get(string);
+            len = (size_t)(rb_enc_left_char_head(str, str + size, str + len, enc) - str);
+        }
+
+        *read = len;
+        memcpy(buf, str, len);
     }
 
     return 1;

gc.c

@@ -79,7 +79,6 @@
 #undef LIST_HEAD /* ccan/list conflicts with BSD-origin sys/queue.h. */
 
 #include "constant.h"
-#include "darray.h"
 #include "debug_counter.h"
 #include "eval_intern.h"
 #include "gc/gc.h"
@@ -1056,7 +1055,15 @@ rb_newobj(rb_execution_context_t *ec, VALUE klass, VALUE flags, shape_id_t shape
 VALUE
 rb_ec_newobj_of(rb_execution_context_t *ec, VALUE klass, VALUE flags, size_t size)
 {
-    return rb_newobj(ec, klass, flags, ROOT_SHAPE_ID, true, size);
+    VALUE type = flags & T_MASK;
+    RUBY_ASSERT(type != T_OBJECT);
+    RUBY_ASSERT(type != T_DATA);
+    RUBY_ASSERT(type != T_CLASS);
+    RUBY_ASSERT(type != T_MODULE);
+    RUBY_ASSERT(type != T_ICLASS);
+    (void)type;
+
+    return rb_newobj(ec, klass, flags, ROOT_SHAPE_ID | SHAPE_ID_LAYOUT_OTHER, true, size);
 }
 
 VALUE
@@ -1068,13 +1075,13 @@ rb_newobj_of_with_shape(VALUE klass, VALUE flags, shape_id_t shape_id, size_t si
 VALUE
 rb_newobj_of(VALUE klass, VALUE flags, size_t size)
 {
-    return rb_newobj(GET_EC(), klass, flags, ROOT_SHAPE_ID, true, size);
+    return rb_newobj(GET_EC(), klass, flags, ROOT_SHAPE_ID | SHAPE_ID_LAYOUT_OTHER, true, size);
 }
 
 static
 VALUE class_allocate_complex_instance(VALUE klass, uint32_t capacity)
 {
-    shape_id_t initial_shape_id = rb_shape_root(rb_gc_heap_id_for_size(sizeof(struct RObject)));
+    shape_id_t initial_shape_id = rb_shape_id_with_robject_layout(rb_shape_root(rb_gc_heap_id_for_size(sizeof(struct RObject))));
     VALUE obj = rb_newobj_of_with_shape(klass, T_OBJECT, initial_shape_id, sizeof(struct RObject));
     rb_obj_init_complex(obj, rb_st_init_numtable_with_size(capacity));
     return obj;
@@ -1099,7 +1106,7 @@ rb_class_allocate_instance(VALUE klass)
 
         // There might be a NEWOBJ tracepoint callback, and it may set fields.
         // So the shape must be passed to `NEWOBJ_OF`.
-        obj = rb_newobj_of_with_shape(klass, T_OBJECT, rb_shape_root(rb_gc_heap_id_for_size(size)), size);
+        obj = rb_newobj_of_with_shape(klass, T_OBJECT, rb_shape_id_with_robject_layout(rb_shape_root(rb_gc_heap_id_for_size(size))), size);
 
         #if RUBY_DEBUG
             VALUE *ptr = ROBJECT_FIELDS(obj);
@@ -1149,7 +1156,7 @@ typed_data_alloc(VALUE klass, VALUE typed_flag, void *datap, const rb_data_type_
     RBIMPL_NONNULL_ARG(type);
     if (klass) rb_data_object_check(klass);
     bool wb_protected = (type->flags & RUBY_FL_WB_PROTECTED) || !type->function.dmark;
-    VALUE obj = rb_newobj(GET_EC(), klass, T_DATA, ROOT_SHAPE_ID, wb_protected, size);
+    VALUE obj = rb_newobj(GET_EC(), klass, T_DATA, ROOT_SHAPE_ID | SHAPE_ID_LAYOUT_RDATA, wb_protected, size);
 
     rb_gc_register_pinning_obj(obj);
 
@@ -4188,7 +4195,11 @@ vm_weak_table_gen_fields_foreach(st_data_t key, st_data_t value, st_data_t data)
         break;
 
       case ST_DELETE:
-        RBASIC_SET_SHAPE_ID((VALUE)key, ROOT_SHAPE_ID);
+        // When we're removing an object from the weak ref table, we need to
+        // set the shape on it so that the GC finalizer won't try to remove
+        // it again.  A "root shape" indicates to the GC that this object
+        // has no fields on it, hence it won't be in the gen fields table.
+        RBASIC_SET_SHAPE_ID((VALUE)key, ROOT_SHAPE_ID | SHAPE_ID_LAYOUT_OTHER);
         return ST_DELETE;
 
       case ST_REPLACE: {

imemo.c

@@ -141,7 +141,11 @@ rb_imemo_fields_new(VALUE owner, shape_id_t shape_id, bool shareable)
     size_t embedded_size = offsetof(struct rb_fields, as.embed) + capa * sizeof(VALUE);
     RUBY_ASSERT(rb_gc_size_allocatable_p(embedded_size));
     VALUE fields = rb_imemo_new(imemo_fields, owner, embedded_size, shareable);
-    RBASIC_SET_SHAPE_ID(fields, shape_id);
+    // imemo fields objects should always have "RObject" layout.  The
+    // layout in the shape describes the layout of the thing on which it is set.
+    // Imemo fields have the same layout as robject, therefore the layout
+    // should reflect that fact.
+    RBASIC_SET_SHAPE_ID(fields, rb_shape_id_with_robject_layout(shape_id));
     RUBY_ASSERT(IMEMO_TYPE_P(fields, imemo_fields));
     return fields;
 }
@@ -152,7 +156,11 @@ rb_imemo_fields_new_complex(VALUE owner, shape_id_t shape_id, size_t capa, bool
     VALUE fields = rb_imemo_new(imemo_fields, owner, sizeof(struct rb_fields), shareable);
     IMEMO_OBJ_FIELDS(fields)->as.complex.table = st_init_numtable_with_size(capa);
     FL_SET_RAW(fields, OBJ_FIELD_HEAP);
-    RBASIC_SET_SHAPE_ID(fields, shape_id);
+    // imemo fields objects should always have "RObject" layout.  The
+    // layout in the shape describes the layout of the thing on which it is set.
+    // Imemo fields have the same layout as robject, therefore the layout
+    // should reflect that fact.
+    RBASIC_SET_SHAPE_ID(fields, rb_shape_id_with_robject_layout(shape_id));
     return fields;
 }
 
@@ -177,7 +185,11 @@ rb_imemo_fields_new_complex_tbl(VALUE owner, shape_id_t shape_id, st_table *tbl,
     VALUE fields = rb_imemo_new(imemo_fields, owner, sizeof(struct rb_fields), shareable);
     IMEMO_OBJ_FIELDS(fields)->as.complex.table = tbl;
     FL_SET_RAW(fields, OBJ_FIELD_HEAP);
-    RBASIC_SET_SHAPE_ID(fields, shape_id);
+    // imemo fields objects should always have "RObject" layout.  The
+    // layout in the shape describes the layout of the thing on which it is set.
+    // Imemo fields have the same layout as robject, therefore the layout
+    // should reflect that fact.
+    RBASIC_SET_SHAPE_ID(fields, rb_shape_id_with_robject_layout(shape_id));
     st_foreach(tbl, imemo_fields_trigger_wb_i, (st_data_t)fields);
     return fields;
 }

internal/gc.h

@@ -124,7 +124,7 @@ struct rb_objspace; /* in vm_core.h */
     T *(var) = (T *)rb_ec_newobj_of((ec), (c), (f), s)
 #define NEWOBJ_OF(var, T, c, f, s) EC_NEWOBJ_OF(var, T, c, f, s, GET_EC())
 #define UNPROTECTED_NEWOBJ_OF(var, T, c, f, s) \
-    T *(var) = (T *)rb_newobj((GET_EC()), (c), (f), 0 /* ROOT_SHAPE_ID */, false, s)
+    T *(var) = (T *)rb_newobj((GET_EC()), (c), (f), ROOT_SHAPE_ID | SHAPE_ID_LAYOUT_OTHER, false, s)
 
 #ifndef RB_GC_OBJECT_METADATA_ENTRY_DEFINED
 # define RB_GC_OBJECT_METADATA_ENTRY_DEFINED

lib/bundler/cli.rb

@@ -274,6 +274,7 @@ def remove(*gems)
     method_option "target-rbconfig", type: :string, banner: "Path to rbconfig.rb for the deployment target platform"
     method_option "without", type: :array, banner: "Exclude gems that are part of the specified named group (removed)."
     method_option "with", type: :array, banner: "Include gems that are part of the specified named group (removed)."
+    method_option "cooldown", type: :numeric, banner: "Only consider gem versions published at least N days ago. Use 0 to disable."
     def install
       %w[clean deployment frozen no-prune path shebang without with].each do |option|
         remembered_flag_deprecation(option)
@@ -324,6 +325,7 @@ def install
     method_option "strict", type: :boolean, banner: "Do not allow any gem to be updated past latest --patch | --minor | --major"
     method_option "conservative", type: :boolean, banner: "Use bundle install conservative update behavior and do not allow shared dependencies to be updated."
     method_option "all", type: :boolean, banner: "Update everything."
+    method_option "cooldown", type: :numeric, banner: "Only consider gem versions published at least N days ago. Use 0 to disable."
     def update(*gems)
       require_relative "cli/update"
       Bundler.settings.temporary(no_install: false) do
@@ -406,6 +408,7 @@ def binstubs(*gems)
     method_option "optimistic", type: :boolean, banner: "Ignored (now default behavior)"
     method_option "pessimistic", type: :boolean, banner: "Adds pessimistic declaration of version to gem"
     method_option "strict", type: :boolean, banner: "Adds strict declaration of version to gem"
+    method_option "cooldown", type: :numeric, banner: "Only consider gem versions published at least N days ago. Use 0 to disable."
     def add(*gems)
       require_relative "cli/add"
       Add.new(options.dup, gems).run
@@ -436,6 +439,7 @@ def add(*gems)
     method_option "filter-patch", type: :boolean, banner: "Only list patch newer versions"
     method_option "parseable", aliases: "--porcelain", type: :boolean, banner: "Use minimal formatting for more parseable output"
     method_option "only-explicit", type: :boolean, banner: "Only list gems specified in your Gemfile, not their dependencies"
+    method_option "cooldown", type: :numeric, banner: "Only consider gem versions published at least N days ago. Use 0 to disable."
     def outdated(*gems)
       require_relative "cli/outdated"
       Outdated.new(options, gems).run

lib/bundler/cli/add.rb

@@ -14,6 +14,9 @@ def initialize(options, gems)
     def run
       Bundler.ui.level = "warn" if options[:quiet]
 
+      Bundler::CLI::Common.validate_cooldown!(options[:cooldown])
+      Bundler.settings.set_command_option_if_given :cooldown, options[:cooldown]
+
       validate_options!
       inject_dependencies
       perform_bundle_install unless options["skip-install"]

lib/bundler/cli/common.rb

@@ -2,6 +2,12 @@
 
 module Bundler
   module CLI::Common
+    def self.validate_cooldown!(value)
+      return if value.nil?
+      return if value.is_a?(Integer) && value >= 0
+      raise InvalidOption, "Expected `--cooldown` to be a non-negative integer, got #{value.inspect}"
+    end
+
     def self.output_post_install_messages(messages)
       return if Bundler.settings["ignore_messages"]
       messages.to_a.each do |name, msg|

lib/bundler/cli/install.rb

@@ -112,6 +112,9 @@ def normalize_settings
 
       Bundler.settings.set_command_option_if_given :jobs, options["jobs"]
 
+      Bundler::CLI::Common.validate_cooldown!(options["cooldown"])
+      Bundler.settings.set_command_option_if_given :cooldown, options["cooldown"]
+
       Bundler.settings.set_command_option_if_given :no_prune, options["no-prune"]
 
       Bundler.settings.set_command_option_if_given :no_install, options["no-install"]

lib/bundler/cli/outdated.rb

@@ -26,6 +26,9 @@ def initialize(options, gems)
     def run
       check_for_deployment_mode!
 
+      Bundler::CLI::Common.validate_cooldown!(options[:cooldown])
+      Bundler.settings.set_command_option_if_given :cooldown, options[:cooldown]
+
       Bundler.definition.validate_runtime!
       current_specs = Bundler.ui.silence { Bundler.definition.resolve }
 
@@ -203,6 +206,10 @@ def print_gem(current_spec, active_spec, dependency, groups)
 
       release_date = release_date_for(active_spec)
       spec_outdated_info += ", released #{release_date}" unless release_date.empty?
+
+      remaining = cooldown_days_remaining(active_spec)
+      spec_outdated_info += ", in cooldown for #{remaining} more day#{"s" if remaining > 1}" if remaining
+
       spec_outdated_info += ")"
 
       output_message = if options[:parseable]
@@ -219,6 +226,8 @@ def print_gem(current_spec, active_spec, dependency, groups)
     def gem_column_for(current_spec, active_spec, dependency, groups)
       current_version = "#{current_spec.version}#{current_spec.git_version}"
       spec_version = "#{active_spec.version}#{active_spec.git_version}"
+      remaining = cooldown_days_remaining(active_spec)
+      spec_version += " (cooldown #{remaining}d)" if remaining
       dependency = dependency.requirement if dependency
 
       ret_val = [active_spec.name, current_version, spec_version, dependency.to_s, groups.to_s]
@@ -227,6 +236,15 @@ def gem_column_for(current_spec, active_spec, dependency, groups)
       ret_val
     end
 
+    def cooldown_days_remaining(spec, now = Time.now)
+      return nil unless spec.respond_to?(:created_at) && spec.created_at
+      return nil unless spec.respond_to?(:remote) && spec.remote
+      days = spec.remote.effective_cooldown
+      return nil if days.nil? || days <= 0
+      remaining = days - ((now - spec.created_at) / 86_400.0)
+      remaining > 0 ? remaining.ceil : nil
+    end
+
     def check_for_deployment_mode!
       return unless Bundler.frozen_bundle?
       suggested_command = if Bundler.settings.locations("frozen").keys.&([:global, :local]).any?

lib/bundler/cli/update.rb

@@ -66,6 +66,8 @@ def run
       opts["force"] = options[:redownload] if options[:redownload]
 
       Bundler.settings.set_command_option_if_given :jobs, opts["jobs"]
+      Bundler::CLI::Common.validate_cooldown!(options[:cooldown])
+      Bundler.settings.set_command_option_if_given :cooldown, options[:cooldown]
 
       Bundler.definition.validate_runtime!
 

lib/bundler/dsl.rb

@@ -117,6 +117,10 @@ def source(source, *args, &blk)
       options = args.last.is_a?(Hash) ? args.pop.dup : {}
       options = normalize_hash(options)
       source = normalize_source(source)
+      cooldown = options["cooldown"]
+      if cooldown && !(cooldown.is_a?(Integer) && cooldown >= 0)
+        raise InvalidOption, "Expected `cooldown` to be a non-negative integer, got #{cooldown.inspect}"
+      end
 
       if options.key?("type")
         options["type"] = options["type"].to_s
@@ -131,9 +135,9 @@ def source(source, *args, &blk)
         source_opts = options.merge("uri" => source)
         with_source(@sources.add_plugin_source(options["type"], source_opts), &blk)
       elsif block_given?
-        with_source(@sources.add_rubygems_source("remotes" => source), &blk)
+        with_source(@sources.add_rubygems_source("remotes" => source, "cooldown" => cooldown), &blk)
       else
-        @sources.add_global_rubygems_remote(source)
+        @sources.add_global_rubygems_remote(source, cooldown: cooldown)
       end
     end