Reintroduce bundler 2.4 (#14995)

* Revert "Revert "Unpin bundler to allow 2.4 (#14894)" (#14942)"

This reverts commit 5e3038a.

* prevent bundler 2.4 from blowing up memory during ./gradlew generatePluginsVersion

for some reason, instances of Bundler::Dsl are retained in
org.jruby.MetaClass. Each Bundler::Dsl will retain Bundler::SourceList
which is > 150MB. This commit sets the internal state of each Dsl's
source list so the large objects can be collected.

The proper fix would be to either reuse a single Dsl object or fix the
code upstream to avoid having Dsl objects retained by org.jruby.MetaClass.

* skip input cloudwatch during generatePluginsVersion

the new bundler 2.4's resolution algorithm seems to struggle in certain
situations, like resolving combination of "logstash-input-cloudwatch" and
"logstash-integration-aws".

Until the issue is solved let's skip it.

build.gradle

@@ -290,7 +290,7 @@ def assemblyDeps = [downloadAndInstallJRuby, assemble] + subprojects.collect {
   it.tasks.findByName("assemble")
 }
 
-def bundlerVersion = "2.3.18"
+def bundlerVersion = "~> 2"
 
 tasks.register("installBundler") {
     dependsOn assemblyDeps

lib/bootstrap/bundler.rb

@@ -163,7 +163,7 @@ def execute_bundler_with_retry(options)
           begin
             execute_bundler(options)
             break
-          rescue ::Bundler::VersionConflict => e
+          rescue ::Bundler::SolveFailure => e
             $stderr.puts("Plugin version conflict, aborting")
             raise(e)
           rescue ::Bundler::GemNotFound => e

rakelib/plugins-metadata.json

@@ -424,6 +424,10 @@
     "default-plugins": true,
     "skip-list": false
   },
+  "logstash-input-cloudwatch": {
+    "default-plugins": false,
+    "skip-list": true
+  },
   "logstash-output-cloudwatch": {
     "default-plugins": false,
     "skip-list": true

rakelib/plugins_docs_dependencies.rake

@@ -126,7 +126,6 @@ class PluginVersionWorking
   end
 
   def try_plugin(plugin, successful_dependencies)
-    Bundler::DepProxy.__clear!
     builder = Bundler::Dsl.new
     gemfile = LogStash::Gemfile.new(File.new(LogStash::Environment::GEMFILE_PATH, "r+")).load
     gemfile.update(plugin)
@@ -136,6 +135,8 @@ class PluginVersionWorking
     definition.resolve_remotely!
     from = PLUGIN_METADATA.fetch(plugin, {}).fetch("default-plugins", false) ? :default : :missing
     extract_versions(definition, successful_dependencies, from)
+    builder.instance_eval { @sources = [] }
+    builder.instance_eval { @dependencies = [] }
   end
 
   def extract_versions(definition, dependencies, from)
@@ -204,14 +205,6 @@ task :generate_plugins_version do
         end
       end
     end
-    DepProxy.class_eval do
-      # Bundler caches it's dep-proxy objects (which contain Gem::Dependency objects) from all resolutions.
-      # The Hash itself continues to grow between dependency resolutions and hold up a lot of memory, to avoid
-      # the issue we expose a way of clear-ing the cached objects before each plugin resolution.
-      def self.__clear!
-        @proxies.clear
-      end
-    end
 
     Fetcher::CompactIndex.class_eval do
       alias_method :__bundle_worker, :bundle_worker

spec/unit/bootstrap/bundler_spec.rb

@@ -88,8 +88,8 @@
 
     context 'abort with an exception' do
       it 'gem conflict' do
-        allow(::Bundler::CLI).to receive(:start).with(bundler_args) { raise ::Bundler::VersionConflict.new('conflict') }
-        expect { subject }.to raise_error(::Bundler::VersionConflict)
+        allow(::Bundler::CLI).to receive(:start).with(bundler_args) { raise ::Bundler::SolveFailure.new('conflict') }
+        expect { subject }.to raise_error(::Bundler::SolveFailure)
       end
 
       it 'gem is not found' do