diff --git a/NEWS.md b/NEWS.md index 33181ad34adeb7..08a2d65e07dbf2 100644 --- a/NEWS.md +++ b/NEWS.md @@ -107,9 +107,10 @@ releases. * test-unit 3.7.7 * 3.7.5 to [3.7.6][test-unit-3.7.6], [3.7.7][test-unit-3.7.7] * net-imap 0.6.4 - * 0.6.2 to [v0.6.3][net-imap-v0.6.3] + * 0.6.2 to [v0.6.3][net-imap-v0.6.3], [v0.6.4][net-imap-v0.6.4] * rbs 4.0.2 * 3.10.0 to [v3.10.1][rbs-v3.10.1], [v3.10.2][rbs-v3.10.2], [v3.10.3][rbs-v3.10.3], [v3.10.4][rbs-v3.10.4], [v4.0.0.dev.5][rbs-v4.0.0.dev.5], [v4.0.0][rbs-v4.0.0], [v4.0.2][rbs-v4.0.2] +* typeprof 0.32.0 * mutex_m 0.3.0 * bigdecimal 4.1.2 * 4.0.1 to [v4.1.0][bigdecimal-v4.1.0], [v4.1.1][bigdecimal-v4.1.1], [v4.1.2][bigdecimal-v4.1.2] @@ -221,6 +222,7 @@ A lot of work has gone into making Ractors more stable, performant, and usable. [test-unit-3.7.6]: https://github.com/test-unit/test-unit/releases/tag/3.7.6 [test-unit-3.7.7]: https://github.com/test-unit/test-unit/releases/tag/3.7.7 [net-imap-v0.6.3]: https://github.com/ruby/net-imap/releases/tag/v0.6.3 +[net-imap-v0.6.4]: https://github.com/ruby/net-imap/releases/tag/v0.6.4 [rbs-v3.10.1]: https://github.com/ruby/rbs/releases/tag/v3.10.1 [rbs-v3.10.2]: https://github.com/ruby/rbs/releases/tag/v3.10.2 [rbs-v3.10.3]: https://github.com/ruby/rbs/releases/tag/v3.10.3 diff --git a/benchmark/string_gsub.yml b/benchmark/string_gsub.yml index 0f964337dd9613..c26e1a649837b8 100644 --- a/benchmark/string_gsub.yml +++ b/benchmark/string_gsub.yml @@ -20,8 +20,19 @@ prelude: | } ESCAPE_PATTERN = Regexp.union(ESCAPED_CHARS.keys) + NO_MATCH_SHARED_STRING = ("a" * 100_000).freeze benchmark: + gsub_no_match_shared: | + str = NO_MATCH_SHARED_STRING.dup + str.gsub!("z", "x") + str + + sub_no_match_shared: | + str = NO_MATCH_SHARED_STRING.dup + str.sub!("z", "x") + str + escape: | str = STR.dup str.gsub!(ESCAPE_PATTERN, ESCAPED_CHARS) diff --git a/benchmark/vm_regexp.yml b/benchmark/vm_regexp.yml index 2aa3d94dbd6bb8..80541332b113bf 100644 --- a/benchmark/vm_regexp.yml +++ b/benchmark/vm_regexp.yml @@ -3,6 +3,12 @@ prelude: | benchmark: vm_regexp: | /hoge/ =~ str + vm_regexp_alternating: | + /hoge/ =~ str + /huge/ =~ str vm_regexp_invert: | str =~ /hoge/ + vm_regexp_invert_alternating: | + str =~ /hoge/ + str =~ /huge/ loop_count: 6000000 diff --git a/depend b/depend index 7faf59f5a44c41..746e62075a99bd 100644 --- a/depend +++ b/depend @@ -5952,6 +5952,7 @@ gc.$(OBJEXT): $(top_srcdir)/internal/numeric.h gc.$(OBJEXT): $(top_srcdir)/internal/object.h gc.$(OBJEXT): $(top_srcdir)/internal/proc.h gc.$(OBJEXT): $(top_srcdir)/internal/rational.h +gc.$(OBJEXT): $(top_srcdir)/internal/re.h gc.$(OBJEXT): $(top_srcdir)/internal/sanitizers.h gc.$(OBJEXT): $(top_srcdir)/internal/serial.h gc.$(OBJEXT): $(top_srcdir)/internal/set_table.h diff --git a/ext/digest/digest.c b/ext/digest/digest.c index bd8d3e815ffe6a..28f60227548400 100644 --- a/ext/digest/digest.c +++ b/ext/digest/digest.c @@ -571,7 +571,7 @@ static rb_digest_metadata_t * get_digest_base_metadata(VALUE klass) { VALUE p; - VALUE obj; + VALUE obj = Qnil; rb_digest_metadata_t *algo; for (p = klass; !NIL_P(p); p = rb_class_superclass(p)) { diff --git a/gc.c b/gc.c index b5eb0dd0fa71f0..191f86ff513d15 100644 --- a/gc.c +++ b/gc.c @@ -100,6 +100,7 @@ #include "internal/object.h" #include "internal/proc.h" #include "internal/rational.h" +#include "internal/re.h" #include "internal/sanitizers.h" #include "internal/struct.h" #include "internal/symbol.h" @@ -1643,17 +1644,19 @@ rb_gc_obj_free(void *objspace, VALUE obj) { struct RMatch *rm = RMATCH(obj); #if USE_DEBUG_COUNTER - if (rm->regs.num_regs >= 8) { + if (rm->num_regs >= 8) { RB_DEBUG_COUNTER_INC(obj_match_ge8); } - else if (rm->regs.num_regs >= 4) { + else if (rm->num_regs >= 4) { RB_DEBUG_COUNTER_INC(obj_match_ge4); } - else if (rm->regs.num_regs >= 1) { + else if (rm->num_regs >= 1) { RB_DEBUG_COUNTER_INC(obj_match_under4); } #endif - onig_region_free(&rm->regs, 0); + if (FL_TEST_RAW(obj, RMATCH_ONIG)) { + onig_region_free(&rm->as.onig, 0); + } SIZED_FREE_N(rm->char_offset, rm->char_offset_num_allocated); RB_DEBUG_COUNTER_INC(obj_match_ptr); @@ -2640,7 +2643,9 @@ rb_obj_memsize_of(VALUE obj) case T_MATCH: { struct RMatch *rm = RMATCH(obj); - size += onig_region_memsize(&rm->regs); + if (FL_TEST_RAW(obj, RMATCH_ONIG)) { + size += onig_region_memsize(&rm->as.onig); + } size += sizeof(struct rmatch_offset) * rm->char_offset_num_allocated; } break; diff --git a/gems/bundled_gems b/gems/bundled_gems index b8fd507be46f1d..f9b0a37ed134ba 100644 --- a/gems/bundled_gems +++ b/gems/bundled_gems @@ -17,7 +17,7 @@ net-smtp 0.5.1 https://github.com/ruby/net-smtp matrix 0.4.3 https://github.com/ruby/matrix prime 0.1.4 https://github.com/ruby/prime rbs 4.0.2 https://github.com/ruby/rbs 36a7e8e38df9efd33db83c3f30f0308bdeb84bd9 -typeprof 0.31.1 https://github.com/ruby/typeprof +typeprof 0.32.0 https://github.com/ruby/typeprof debug 1.11.1 https://github.com/ruby/debug 9dc2024a5a05116b3d38afbc5579d9503d8913f3 racc 1.8.1 https://github.com/ruby/racc mutex_m 0.3.0 https://github.com/ruby/mutex_m @@ -37,7 +37,7 @@ ostruct 0.6.3 https://github.com/ruby/ostruct pstore 0.2.1 https://github.com/ruby/pstore benchmark 0.5.0 https://github.com/ruby/benchmark logger 1.7.0 https://github.com/ruby/logger -rdoc 7.2.0 https://github.com/ruby/rdoc 1f93543615928b6d45357432f16ec6006e2d8b1e +rdoc 7.2.0 https://github.com/ruby/rdoc bc0baee787609f2205e8f1103f3e1d36b923184a win32ole 1.9.3 https://github.com/ruby/win32ole irb 1.18.0 https://github.com/ruby/irb reline 0.6.3 https://github.com/ruby/reline diff --git a/include/ruby/internal/core/rmatch.h b/include/ruby/internal/core/rmatch.h index 7db9a8f196c004..ae937555313a71 100644 --- a/include/ruby/internal/core/rmatch.h +++ b/include/ruby/internal/core/rmatch.h @@ -91,21 +91,40 @@ struct RMatch { */ VALUE regexp; /* RRegexp */ - /** - * "Registers" of a match. This is a quasi-opaque struct that holds - * execution result of a match. Roughly resembles `$~`. - */ - struct re_registers regs; + /** Number of ::rmatch_offset that ::rmatch::char_offset holds. */ + int char_offset_num_allocated; - /** Number of ::rmatch_offset that ::rmatch::char_offset holds. */ - int char_offset_num_allocated; + /** Capture group offsets, in C array. */ + struct rmatch_offset *char_offset; - /** Capture group offsets, in C array. */ - struct rmatch_offset *char_offset; + /** Number of capture-group registers. */ + int num_regs; + + /** Capacity of `as.embed`, in OnigPosition slots. */ + int capa; + + /** + * "Registers" of a match. This is a quasi-opaque struct that holds + * execution result of a match. Roughly resembles `$~`. + */ + union { + OnigPosition embed[1]; + struct re_registers onig; + } as; }; -RBIMPL_ATTR_PURE_UNLESS_DEBUG() -RBIMPL_ATTR_ARTIFICIAL() +RBIMPL_SYMBOL_EXPORT_BEGIN() +/** + * @private + * + * Converts an embedded match to onig form. This is an implementation + * detail of #RMATCH_REGS. People don't use it directly. + * + * @param[out] match A match object, possibly in embedded form. + */ +void rb_match_ensure_onig(VALUE match); +RBIMPL_SYMBOL_EXPORT_END() + /** * Queries the raw ::re_registers. * @@ -131,7 +150,8 @@ static inline struct re_registers * RMATCH_REGS(VALUE match) { RBIMPL_ASSERT_TYPE(match, RUBY_T_MATCH); - return &RMATCH(match)->regs; + rb_match_ensure_onig(match); + return &RMATCH(match)->as.onig; } #endif /* RBIMPL_RMATCH_H */ diff --git a/internal/class.h b/internal/class.h index 45f2fde5f8dd93..e02e1a408bdf18 100644 --- a/internal/class.h +++ b/internal/class.h @@ -297,7 +297,7 @@ RCLASS_PRIME_CLASSEXT_WRITABLE_P(VALUE klass) { VM_ASSERT(klass != 0, "klass should be a valid object"); VM_ASSERT_BOXABLE_TYPE(klass); - return FL_TEST(klass, RCLASS_PRIME_CLASSEXT_WRITABLE); + return FL_TEST_RAW(klass, RCLASS_PRIME_CLASSEXT_WRITABLE); } static inline void @@ -306,10 +306,10 @@ RCLASS_SET_PRIME_CLASSEXT_WRITABLE(VALUE klass, bool writable) VM_ASSERT(klass != 0, "klass should be a valid object"); VM_ASSERT_BOXABLE_TYPE(klass); if (writable) { - FL_SET(klass, RCLASS_PRIME_CLASSEXT_WRITABLE); + FL_SET_RAW(klass, RCLASS_PRIME_CLASSEXT_WRITABLE); } else { - FL_UNSET(klass, RCLASS_PRIME_CLASSEXT_WRITABLE); + FL_UNSET_RAW(klass, RCLASS_PRIME_CLASSEXT_WRITABLE); } } diff --git a/internal/re.h b/internal/re.h index aa1c93f64275dd..b0acac6033ec89 100644 --- a/internal/re.h +++ b/internal/re.h @@ -12,16 +12,28 @@ #include "ruby/ruby.h" /* for VALUE */ #include "ruby/re.h" /* for struct RMatch and struct re_registers */ +#define RMATCH_ONIG FL_USER1 + static inline OnigPosition * RMATCH_BEG_PTR(VALUE match) { - return RMATCH(match)->regs.beg; + if (FL_TEST_RAW(match, RMATCH_ONIG)) { + return RMATCH(match)->as.onig.beg; + } + else { + return &RMATCH(match)->as.embed[0]; + } } static inline OnigPosition * RMATCH_END_PTR(VALUE match) { - return RMATCH(match)->regs.end; + if (FL_TEST_RAW(match, RMATCH_ONIG)) { + return RMATCH(match)->as.onig.end; + } + else { + return &RMATCH(match)->as.embed[RMATCH(match)->num_regs]; + } } static inline long @@ -39,7 +51,7 @@ RMATCH_END(VALUE match, int i) static inline int RMATCH_NREGS(VALUE match) { - return RMATCH(match)->regs.num_regs; + return RMATCH(match)->num_regs; } /* re.c */ @@ -48,6 +60,7 @@ VALUE rb_reg_compile(VALUE str, int options, const char *sourcefile, int sourcel VALUE rb_reg_check_preprocess(VALUE); long rb_reg_search0(VALUE, VALUE, long, int, int, VALUE *); VALUE rb_reg_match_p(VALUE re, VALUE str, long pos); +VALUE rb_reg_regsub_match(VALUE str, VALUE src, VALUE match); bool rb_reg_start_with_p(VALUE re, VALUE str); VALUE rb_reg_hash(VALUE re); VALUE rb_reg_equal(VALUE re1, VALUE re2); diff --git a/re.c b/re.c index 317790fba80e5e..b8b5963d725b7a 100644 --- a/re.c +++ b/re.c @@ -37,9 +37,6 @@ VALUE rb_eRegexpError, rb_eRegexpTimeoutError; typedef char onig_errmsg_buffer[ONIG_MAX_ERROR_MESSAGE_LEN]; #define errcpy(err, msg) strlcpy((err), (msg), ONIG_MAX_ERROR_MESSAGE_LEN) -#define BEG(no) (regs->beg[(no)]) -#define END(no) (regs->end[(no)]) - #if 'a' == 97 /* it's ascii */ static const char casetable[] = { '\000', '\001', '\002', '\003', '\004', '\005', '\006', '\007', @@ -965,14 +962,33 @@ make_regexp(const char *s, long len, rb_encoding *enc, int flags, onig_errmsg_bu VALUE rb_cMatch; static VALUE -match_alloc(VALUE klass) +match_alloc_n(VALUE klass, int num_regs) { - size_t alloc_size = sizeof(struct RMatch); - NEWOBJ_OF(match, struct RMatch, klass, T_MATCH, alloc_size); - memset(((char *)match) + sizeof(struct RBasic), 0, sizeof(struct RMatch) - sizeof(struct RBasic)); + int capa = num_regs * 2; + size_t alloc_size = offsetof(struct RMatch, as) + sizeof(OnigPosition) * capa; + if (alloc_size < sizeof(struct RMatch)) { + alloc_size = sizeof(struct RMatch); + } + + VALUE flags = T_MATCH; + if (!rb_gc_size_allocatable_p(alloc_size)) { + alloc_size = sizeof(struct RMatch); + flags |= RMATCH_ONIG; + capa = 0; + } + + NEWOBJ_OF(match, struct RMatch, klass, flags, alloc_size); + memset(((char *)match) + sizeof(struct RBasic), 0, alloc_size - sizeof(struct RBasic)); + match->capa = capa; return (VALUE)match; } +static VALUE +match_alloc(VALUE klass) +{ + return match_alloc_n(klass, 0); +} + int rb_reg_region_copy(struct re_registers *to, const struct re_registers *from) { @@ -984,6 +1000,54 @@ rb_reg_region_copy(struct re_registers *to, const struct re_registers *from) return ONIGERR_MEMORY; } +static void +match_to_onig(VALUE match, int num_regs, const OnigPosition *src_beg, const OnigPosition *src_end) +{ + struct RMatch *rm = RMATCH(match); + struct re_registers tmp = {0}; + if (onig_region_resize(&tmp, num_regs)) { + rb_memerror(); + } + memcpy(tmp.beg, src_beg, num_regs * sizeof(OnigPosition)); + memcpy(tmp.end, src_end, num_regs * sizeof(OnigPosition)); + rm->as.onig = tmp; + FL_SET_RAW(match, RMATCH_ONIG); +} + +void +rb_match_ensure_onig(VALUE match) +{ + if (FL_TEST_RAW(match, RMATCH_ONIG)) return; + struct RMatch *rm = RMATCH(match); + int n = rm->num_regs; + match_to_onig(match, n, &rm->as.embed[0], &rm->as.embed[n]); +} + +/* Replace `match`'s registers with a copy of (num_regs, beg, end). If the + * data does not fit in the embed form, the match is evicted to onig form. + * Raises on OOM. */ +static void +match_set_regs(VALUE match, int num_regs, const OnigPosition *beg, const OnigPosition *end) +{ + struct RMatch *rm = RMATCH(match); + + if (FL_TEST_RAW(match, RMATCH_ONIG)) { + if (onig_region_resize(&rm->as.onig, num_regs)) { + rb_memerror(); + } + memcpy(rm->as.onig.beg, beg, num_regs * sizeof(OnigPosition)); + memcpy(rm->as.onig.end, end, num_regs * sizeof(OnigPosition)); + } + else if (num_regs * 2 <= rm->capa) { + memcpy(&rm->as.embed[0], beg, num_regs * sizeof(OnigPosition)); + memcpy(&rm->as.embed[num_regs], end, num_regs * sizeof(OnigPosition)); + } + else { + match_to_onig(match, num_regs, beg, end); + } + rm->num_regs = num_regs; +} + typedef struct { long byte_pos; long char_pos; @@ -1089,16 +1153,15 @@ match_init_copy(VALUE obj, VALUE orig) RB_OBJ_WRITE(obj, &rm->str, RMATCH(orig)->str); RB_OBJ_WRITE(obj, &rm->regexp, RMATCH(orig)->regexp); - if (rb_reg_region_copy(&rm->regs, RMATCH_REGS(orig))) - rb_memerror(); + match_set_regs(obj, RMATCH_NREGS(orig), RMATCH_BEG_PTR(orig), RMATCH_END_PTR(orig)); if (RMATCH(orig)->char_offset_num_allocated) { - if (rm->char_offset_num_allocated < rm->regs.num_regs) { - SIZED_REALLOC_N(rm->char_offset, struct rmatch_offset, rm->regs.num_regs, rm->char_offset_num_allocated); - rm->char_offset_num_allocated = rm->regs.num_regs; + if (rm->char_offset_num_allocated < rm->num_regs) { + SIZED_REALLOC_N(rm->char_offset, struct rmatch_offset, rm->num_regs, rm->char_offset_num_allocated); + rm->char_offset_num_allocated = rm->num_regs; } MEMCPY(rm->char_offset, RMATCH(orig)->char_offset, - struct rmatch_offset, rm->regs.num_regs); + struct rmatch_offset, rm->num_regs); RB_GC_GUARD(orig); } @@ -1180,7 +1243,7 @@ match_size(VALUE match) return INT2FIX(RMATCH_NREGS(match)); } -static int name_to_backref_number(const struct re_registers *, VALUE, const char*, const char*); +static int match_name_to_backref_number(VALUE match, VALUE name); NORETURN(static void name_to_backref_error(VALUE name)); static void @@ -1200,12 +1263,8 @@ backref_number_check(VALUE match, int i) static int match_backref_number(VALUE match, VALUE backref) { - const char *name; int num; - struct re_registers *regs = RMATCH_REGS(match); - VALUE regexp = RMATCH(match)->regexp; - match_check(match); if (SYMBOL_P(backref)) { backref = rb_sym2str(backref); @@ -1213,9 +1272,8 @@ match_backref_number(VALUE match, VALUE backref) else if (!RB_TYPE_P(backref, T_STRING)) { return NUM2INT(backref); } - name = StringValueCStr(backref); - num = name_to_backref_number(regs, regexp, name, name + RSTRING_LEN(backref)); + num = match_name_to_backref_number(match, backref); if (num < 1) { name_to_backref_error(backref); @@ -1489,6 +1547,17 @@ rb_match_count(VALUE match) return RMATCH_NREGS(match); } +static VALUE +match_alloc_or_reuse(VALUE existing, int num_regs) +{ + if (!NIL_P(existing) && + !FL_TEST(existing, MATCH_BUSY) && + RMATCH(existing)->capa >= num_regs * 2) { + return existing; + } + return match_alloc_n(rb_cMatch, num_regs); +} + static void match_set_string(VALUE m, VALUE string, long pos, long len) { @@ -1496,19 +1565,14 @@ match_set_string(VALUE m, VALUE string, long pos, long len) RB_OBJ_WRITE(match, &match->str, string); RB_OBJ_WRITE(match, &match->regexp, Qnil); - int err = onig_region_resize(&match->regs, 1); - if (err) rb_memerror(); - match->regs.beg[0] = pos; - match->regs.end[0] = pos + len; + OnigPosition beg = pos, end = pos + len; + match_set_regs(m, 1, &beg, &end); } VALUE rb_backref_set_string(VALUE string, long pos, long len) { - VALUE match = rb_backref_get(); - if (NIL_P(match) || FL_TEST(match, MATCH_BUSY)) { - match = match_alloc(rb_cMatch); - } + VALUE match = match_alloc_or_reuse(rb_backref_get(), 1); match_set_string(match, string, pos, len); rb_backref_set(match); return match; @@ -1689,8 +1753,6 @@ rb_reg_onig_match(VALUE re, VALUE str, } if (result < 0) { - onig_region_free(regs, 0); - switch (result) { case ONIG_MISMATCH: break; @@ -1775,37 +1837,35 @@ rb_reg_search_set_match(VALUE re, VALUE str, long pos, int reverse, int set_back .pos = pos, .range = reverse ? 0 : len, }; - struct re_registers regs = {0}; + + rb_reg_check(re); + + /* Stack-backed regs sized to max(num_mem+1, ONIG_NREGION) so + * onig_region_resize_clear takes its no-op branch. */ + int n = RREGEXP_PTR(re)->num_mem + 1; + int cap = n < ONIG_NREGION ? ONIG_NREGION : n; + VALUE regs_buf; + OnigPosition *buf = ALLOCV_N(OnigPosition, regs_buf, (size_t)cap * 2); + struct re_registers regs = { + .allocated = cap, + .num_regs = 0, + .beg = buf, + .end = buf + cap, + }; OnigPosition result = rb_reg_onig_match(re, str, reg_onig_search, &args, ®s); if (result == ONIG_MISMATCH) { + ALLOCV_END(regs_buf); rb_backref_set(Qnil); return ONIG_MISMATCH; } - VALUE match = Qnil; - if (set_match) { - match = *set_match; - } - - if (NIL_P(match)) { - match = rb_backref_get(); - } - - if (!NIL_P(match) && FL_TEST(match, MATCH_BUSY)) { - match = Qnil; - } - - if (NIL_P(match)) { - match = match_alloc(rb_cMatch); - } - else { - onig_region_free(&RMATCH(match)->regs, false); - } + VALUE existing = (set_match && !NIL_P(*set_match)) ? *set_match : rb_backref_get(); + VALUE match = match_alloc_or_reuse(existing, regs.num_regs); - struct RMatch *rm = RMATCH(match); - rm->regs = regs; + match_set_regs(match, regs.num_regs, regs.beg, regs.end); + ALLOCV_END(regs_buf); if (set_backref_str) { RB_OBJ_WRITE(match, &RMATCH(match)->str, rb_str_new4(str)); @@ -1857,18 +1917,29 @@ reg_onig_match(regex_t *reg, VALUE str, struct re_registers *regs, void *_) bool rb_reg_start_with_p(VALUE re, VALUE str) { - VALUE match = rb_backref_get(); - if (NIL_P(match) || FL_TEST(match, MATCH_BUSY)) { - match = match_alloc(rb_cMatch); - } + rb_reg_check(re); - struct re_registers *regs = RMATCH_REGS(match); + int n = RREGEXP_PTR(re)->num_mem + 1; + int cap = n < ONIG_NREGION ? ONIG_NREGION : n; + VALUE regs_buf; + OnigPosition *buf = ALLOCV_N(OnigPosition, regs_buf, (size_t)cap * 2); + struct re_registers regs = { + .allocated = cap, + .num_regs = 0, + .beg = buf, + .end = buf + cap, + }; - if (rb_reg_onig_match(re, str, reg_onig_match, NULL, regs) == ONIG_MISMATCH) { + if (rb_reg_onig_match(re, str, reg_onig_match, NULL, ®s) == ONIG_MISMATCH) { + ALLOCV_END(regs_buf); rb_backref_set(Qnil); return false; } + VALUE match = match_alloc_or_reuse(rb_backref_get(), regs.num_regs); + match_set_regs(match, regs.num_regs, regs.beg, regs.end); + ALLOCV_END(regs_buf); + RB_OBJ_WRITE(match, &RMATCH(match)->str, rb_str_new4(str)); RB_OBJ_WRITE(match, &RMATCH(match)->regexp, re); rb_backref_set(match); @@ -2123,7 +2194,27 @@ name_to_backref_number(const struct re_registers *regs, VALUE regexp, const char name_to_backref_number((regs), (re), (name_ptr), (name_end))) static int -namev_to_backref_number(const struct re_registers *regs, VALUE re, VALUE name) +match_name_to_backref_number(VALUE match, VALUE name) +{ + VALUE regexp = RMATCH(match)->regexp; + if (NIL_P(regexp)) return -1; + + int *nums; + int n = onig_name_to_group_numbers(RREGEXP_PTR(regexp), + (const unsigned char *)RSTRING_PTR(name), + (const unsigned char *)RSTRING_END(name), &nums); + if (n < 0) return n; + if (n == 0) return ONIGERR_PARSER_BUG; + if (n == 1) return nums[0]; + for (int i = n - 1; i >= 0; i--) { + if (RMATCH_BEG(match, nums[i]) != ONIG_REGION_NOTPOS) + return nums[i]; + } + return nums[n - 1]; +} + +static int +namev_to_backref_number(VALUE match, VALUE name) { int num; @@ -2133,8 +2224,14 @@ namev_to_backref_number(const struct re_registers *regs, VALUE re, VALUE name) else if (!RB_TYPE_P(name, T_STRING)) { return -1; } - num = NAME_TO_NUMBER(regs, re, name, - RSTRING_PTR(name), RSTRING_END(name)); + + VALUE re = RMATCH(match)->regexp; + if (NIL_P(re) || !rb_enc_compatible(RREGEXP_SRC(re), name)) { + num = 0; + } + else { + num = match_name_to_backref_number(match, name); + } if (num < 1) { name_to_backref_error(name); } @@ -2228,7 +2325,7 @@ match_aref(int argc, VALUE *argv, VALUE match) return rb_reg_nth_match(FIX2INT(idx), match); } else { - int num = namev_to_backref_number(RMATCH_REGS(match), RMATCH(match)->regexp, idx); + int num = namev_to_backref_number(match, idx); if (num >= 0) { return rb_reg_nth_match(num, match); } @@ -2298,7 +2395,7 @@ match_values_at(int argc, VALUE *argv, VALUE match) rb_ary_push(result, rb_reg_nth_match(FIX2INT(argv[i]), match)); } else { - int num = namev_to_backref_number(RMATCH_REGS(match), RMATCH(match)->regexp, argv[i]); + int num = namev_to_backref_number(match, argv[i]); if (num >= 0) { rb_ary_push(result, rb_reg_nth_match(num, match)); } @@ -2500,8 +2597,7 @@ match_deconstruct_keys(VALUE match, VALUE keys) name = rb_sym2str(key); - int num = NAME_TO_NUMBER(RMATCH_REGS(match), RMATCH(match)->regexp, RMATCH(match)->regexp, - RSTRING_PTR(name), RSTRING_END(name)); + int num = match_name_to_backref_number(match, name); if (num >= 0) { rb_hash_aset(h, key, rb_reg_nth_match(num, match)); @@ -3627,7 +3723,7 @@ match_equal(VALUE match1, VALUE match2) static VALUE match_integer_at(int argc, VALUE *argv, VALUE match) { - const struct re_registers *regs = RMATCH_REGS(match_check(match)); + match_check(match); int base = 10; VALUE idx; @@ -3637,7 +3733,7 @@ match_integer_at(int argc, VALUE *argv, VALUE match) if (FIXNUM_P(idx = argv[0])) { nth = NUM2INT(idx); } - else if ((nth = namev_to_backref_number(regs, RMATCH(match)->regexp, idx)) < 0) { + else if ((nth = namev_to_backref_number(match, idx)) < 0) { name_to_backref_error(idx); } @@ -4504,8 +4600,8 @@ rb_reg_init_copy(VALUE copy, VALUE re) return reg_copy(copy, re); } -VALUE -rb_reg_regsub(VALUE str, VALUE src, struct re_registers *regs, VALUE regexp) +static VALUE +do_regsub(VALUE str, VALUE src, VALUE regexp, int num_regs, const OnigPosition *beg, const OnigPosition *end) { VALUE val = 0; char *p, *s, *e; @@ -4572,7 +4668,13 @@ rb_reg_regsub(VALUE str, VALUE src, struct re_registers *regs, VALUE regexp) if (name_end < e) { VALUE n = rb_str_subseq(str, (long)(name - RSTRING_PTR(str)), (long)(name_end - name)); - if ((no = NAME_TO_NUMBER(regs, regexp, n, name, name_end)) < 1) { + struct re_registers tmp = { + .allocated = num_regs, + .num_regs = num_regs, + .beg = (OnigPosition *)beg, + .end = (OnigPosition *)end, + }; + if ((no = NAME_TO_NUMBER(&tmp, regexp, n, name, name_end)) < 1) { name_to_backref_error(n); } p = s = name_end + clen; @@ -4592,16 +4694,16 @@ rb_reg_regsub(VALUE str, VALUE src, struct re_registers *regs, VALUE regexp) break; case '`': - rb_enc_str_buf_cat(val, RSTRING_PTR(src), BEG(0), src_enc); + rb_enc_str_buf_cat(val, RSTRING_PTR(src), beg[0], src_enc); continue; case '\'': - rb_enc_str_buf_cat(val, RSTRING_PTR(src)+END(0), RSTRING_LEN(src)-END(0), src_enc); + rb_enc_str_buf_cat(val, RSTRING_PTR(src)+end[0], RSTRING_LEN(src)-end[0], src_enc); continue; case '+': - no = regs->num_regs-1; - while (BEG(no) == -1 && no > 0) no--; + no = num_regs-1; + while (beg[no] == -1 && no > 0) no--; if (no == 0) continue; break; @@ -4615,9 +4717,9 @@ rb_reg_regsub(VALUE str, VALUE src, struct re_registers *regs, VALUE regexp) } if (no >= 0) { - if (no >= regs->num_regs) continue; - if (BEG(no) == -1) continue; - rb_enc_str_buf_cat(val, RSTRING_PTR(src)+BEG(no), END(no)-BEG(no), src_enc); + if (no >= num_regs) continue; + if (beg[no] == -1) continue; + rb_enc_str_buf_cat(val, RSTRING_PTR(src)+beg[no], end[no]-beg[no], src_enc); } } @@ -4627,6 +4729,20 @@ rb_reg_regsub(VALUE str, VALUE src, struct re_registers *regs, VALUE regexp) } return val; +#undef ASCGET +} + +VALUE +rb_reg_regsub(VALUE str, VALUE src, struct re_registers *regs, VALUE regexp) +{ + return do_regsub(str, src, regexp, regs->num_regs, regs->beg, regs->end); +} + +VALUE +rb_reg_regsub_match(VALUE str, VALUE src, VALUE match) +{ + return do_regsub(str, src, RMATCH(match)->regexp, + RMATCH_NREGS(match), RMATCH_BEG_PTR(match), RMATCH_END_PTR(match)); } static VALUE diff --git a/spec/ruby/security/cve_2019_8322_spec.rb b/spec/ruby/security/cve_2019_8322_spec.rb index c2614b10d54721..dbc273f313ed69 100644 --- a/spec/ruby/security/cve_2019_8322_spec.rb +++ b/spec/ruby/security/cve_2019_8322_spec.rb @@ -1,6 +1,6 @@ require_relative '../spec_helper' -ruby_version_is ""..."4.1" do +ruby_version_is ""..."4.0" do require 'yaml' require 'rubygems' diff --git a/string.c b/string.c index e4de2f0e66a786..c72e2a8e36b17b 100644 --- a/string.c +++ b/string.c @@ -6258,12 +6258,10 @@ rb_str_sub_bang(int argc, VALUE *argv, VALUE str) int cr = ENC_CODERANGE(str); long beg0, end0; VALUE match, match0 = Qnil; - struct re_registers *regs; char *p, *rp; long len, rlen; match = rb_backref_get(); - regs = RMATCH_REGS(match); if (RB_TYPE_P(pat, T_STRING)) { beg0 = beg; end0 = beg0 + RSTRING_LEN(pat); @@ -6289,7 +6287,7 @@ rb_str_sub_bang(int argc, VALUE *argv, VALUE str) rb_check_frozen(str); } else { - repl = rb_reg_regsub(repl, str, regs, RB_TYPE_P(pat, T_STRING) ? Qnil : pat); + repl = rb_reg_regsub_match(repl, str, match); } enc = rb_enc_compatible(str, repl); @@ -6397,6 +6395,7 @@ str_gsub(int argc, VALUE *argv, VALUE str, int bang) if (bang) return Qnil; /* no match, no substitution */ return str_duplicate(rb_cString, str); } + if (bang) str_modify_keep_cr(str); offset = 0; blen = RSTRING_LEN(str) + 30; /* len + margin */ @@ -6409,7 +6408,6 @@ str_gsub(int argc, VALUE *argv, VALUE str, int bang) ENC_CODERANGE_SET(dest, rb_enc_asciicompat(str_enc) ? ENC_CODERANGE_7BIT : ENC_CODERANGE_VALID); do { - struct re_registers *regs = RMATCH_REGS(match); if (RB_TYPE_P(pat, T_STRING)) { beg0 = beg; end0 = beg0 + RSTRING_LEN(pat); @@ -6446,7 +6444,7 @@ str_gsub(int argc, VALUE *argv, VALUE str, int bang) } } else if (need_backref_str) { - val = rb_reg_regsub(repl, str, regs, RB_TYPE_P(pat, T_STRING) ? Qnil : pat); + val = rb_reg_regsub_match(repl, str, match); if (need_backref_str < 0) { need_backref_str = val != repl; } @@ -6518,7 +6516,7 @@ str_gsub(int argc, VALUE *argv, VALUE str, int bang) static VALUE rb_str_gsub_bang(int argc, VALUE *argv, VALUE str) { - str_modify_keep_cr(str); + str_modifiable(str); return str_gsub(argc, argv, str, 1); } diff --git a/test/ruby/test_backtrace.rb b/test/ruby/test_backtrace.rb index dad7dfcb558471..332d76c58e1596 100644 --- a/test/ruby/test_backtrace.rb +++ b/test/ruby/test_backtrace.rb @@ -191,6 +191,16 @@ def test_each_backtrace_location assert_equal(cl.map(&:to_s), ary.map(&:to_s)) end + def test_each_caller_location_single_cfunc_frame + assert_normal_exit <<~'RUBY' + tap { Thread.each_caller_location(1, 1) { |loc| loc.label } } + RUBY + + cl = nil; ary = [] + tap { cl = caller_locations(1, 1); Thread.each_caller_location(1, 1) { |x| ary << x } } + assert_equal(cl.map(&:to_s), ary.map(&:to_s)) + end + def test_caller_locations_first_label def self.label caller_locations.first.label diff --git a/test/ruby/test_regexp.rb b/test/ruby/test_regexp.rb index b519872d6718aa..805c57b47219c5 100644 --- a/test/ruby/test_regexp.rb +++ b/test/ruby/test_regexp.rb @@ -1011,6 +1011,18 @@ def test_regsub_no_memory_leak end; end + def test_regsub_no_memory_leak_many_captures + assert_no_memory_leak([], "#{<<~"begin;"}", "#{<<~"end;"}", rss: true) + code = proc do + "aaaaaaaaaaa".gsub(/(a)(b)?(c)?(d)?(e)?(f)?(g)?(h)?/, "") + end + + 1_000.times(&code) + begin; + 100_000.times(&code) + end; + end + def test_ignorecase v = assert_deprecated_warning(/variable \$= is no longer effective/) { $= } assert_equal(false, v) diff --git a/vm_backtrace.c b/vm_backtrace.c index ea876b150282f2..85a06586120030 100644 --- a/vm_backtrace.c +++ b/vm_backtrace.c @@ -751,7 +751,7 @@ rb_ec_partial_backtrace_object(const rb_execution_context_t *ec, long start_fram bt_backpatch_loc(backpatch_counter, loc, CFP_ISEQ(cfp), CFP_PC(cfp)); RB_OBJ_WRITTEN(btobj, Qundef, CFP_ISEQ(cfp)); if (do_yield) { - bt_yield_loc(loc - backpatch_counter, backpatch_counter, btobj); + bt_yield_loc(loc - backpatch_counter + 1, backpatch_counter, btobj); } break; }