Fix and add regexp tests

test/ruby/test_regexp.rb

@@ -1589,7 +1589,7 @@ def test_s_timeout
       t = Time.now
       assert_raise_with_message(Regexp::TimeoutError, "regexp match timeout") do
         # A typical ReDoS case
-        /^(a*)*$/ =~ "a" * 1000000 + "x"
+        /^(a*)*\1$/ =~ "a" * 1000000 + "x"
       end
       t = Time.now - t
 
@@ -1631,7 +1631,7 @@ def per_instance_redos_test(global_timeout, per_instance_timeout, expected_timeo
 
       Regexp.timeout = global_timeout
 
-      re = Regexp.new("^a*b?a*$", timeout: per_instance_timeout)
+      re = Regexp.new("^(a*)\\1b?a*$", timeout: per_instance_timeout)
       assert_equal(per_instance_timeout, re.timeout)
 
       t = Time.now
@@ -1673,4 +1673,24 @@ def test_timeout_corner_cases
       assert_raise(ArgumentError) { Regexp.new("foo", timeout: -1) }
     end;
   end
+
+  def test_cache_optimization_exponential
+    assert_separately([], "#{<<-"begin;"}\n#{<<-'end;'}")
+    begin;
+      timeout = EnvUtil.apply_timeout_scale(0.2)
+      Regexp.timeout = timeout
+
+      assert_nil(/^(a*)*$/ =~ "a" * 1000000 + "x")
+    end;
+  end
+
+  def test_cache_optimization_square
+    assert_separately([], "#{<<-"begin;"}\n#{<<-'end;'}")
+    begin;
+      timeout = EnvUtil.apply_timeout_scale(0.2)
+      Regexp.timeout = timeout
+
+      assert_nil(/^a*b?a*$/ =~ "a" * 1000000 + "x")
+    end;
+  end
 end