Package elastic_integration plugin. (#15769)

* Exclude plugins feature in OSS distributions.

* Set elastic_integration plugin default.

* Remove non-OSS plugins after installing default plugins.

* Testing local can't find gem bundler (= 2.3.26) issue.

* Include extract non-OSS plugins logic indocker build operations.

* Only default plugins can be excluded from OSS distros.

* Simplification: instead conditional check, use intersection to make OSS exlucluded plugin list.

* Gem and specification files still stay after removing the plugin. This change removes the stayed files.

* Rename oss-exclude to skip-oss to align namings with other params.

* Make intersection method simpler.

* [Test] Temporary excluding elastic integration plugin from default plugin list.

* Sets elastic_integration plugin default back. When removing locally installed gems, Gem::Specification doesn't recognize the gem. We have Bundle::setup in the removal logic but it is causing an issue when we re-use the bundle.

* Test the build order, remove plugin from cache logic seems invalid since we don't pack the cache.

lib/pluginmanager/remove.rb

@@ -25,12 +25,7 @@ class LogStash::PluginManager::Remove < LogStash::PluginManager::Command
   def execute
     signal_error("File #{LogStash::Environment::GEMFILE_PATH} does not exist or is not writable, aborting") unless File.writable?(LogStash::Environment::GEMFILE_PATH)
 
-    ##
-    # Need to setup the bundler status to enable uninstall of plugins
-    # installed as local_gems, otherwise gem:specification is not
-    # finding the plugins
-    ##
-    LogStash::Bundler.setup!({:without => [:build, :development]})
+    LogStash::Bundler.prepare({:without => [:build, :development]})
 
     if LogStash::PluginManager::ALIASES.has_key?(plugin)
       unless LogStash::PluginManager.installed_plugin?(plugin, gemfile)
@@ -49,9 +44,22 @@ def execute
       signal_error("This plugin is already provided by '#{integration_plugin.name}' so it can't be removed individually")
     end
 
-    # make sure this is an installed plugin and present in Gemfile.
-    # it is not possible to uninstall a dependency not listed in the Gemfile, for example a dependent codec
-    signal_error("This plugin has not been previously installed") unless LogStash::PluginManager.installed_plugin?(plugin, gemfile)
+    not_installed_message = "This plugin has not been previously installed"
+    plugin_gem_spec = LogStash::PluginManager.find_plugins_gem_specs(plugin).any?
+    if plugin_gem_spec
+      # make sure this is an installed plugin and present in Gemfile.
+      # it is not possible to uninstall a dependency not listed in the Gemfile, for example a dependent codec
+      signal_error(not_installed_message) unless LogStash::PluginManager.installed_plugin?(plugin, gemfile)
+    else
+      # locally installed plugins are not recoginized by ::Gem::Specification
+      # we may ::Bundler.setup to reload but it resets all dependencies so we get error message for future bundler operations
+      # error message: `Bundler::GemNotFound: Could not find rubocop-1.60.2... in locally installed gems`
+      # instead we make sure Gemfile has a definition and ::Gem::Specification recognizes local gem
+      signal_error(not_installed_message) unless !!gemfile.find(plugin)
+
+      local_gem = gemfile.locally_installed_gems.detect { |local_gem| local_gem.name == plugin }
+      signal_error(not_installed_message) unless local_gem
+    end
 
     exit(1) unless ::Bundler::LogstashUninstall.uninstall!(plugin)
     LogStash::Bundler.genericize_platform

rakelib/artifacts.rake

@@ -193,7 +193,7 @@ namespace "artifact" do
   end
 
   desc "Build all (jdk bundled and not) OSS tar.gz and zip of default logstash plugins with all dependencies"
-  task "archives_oss" => ["prepare", "generate_build_metadata"] do
+  task "archives_oss" => ["prepare-oss", "generate_build_metadata"] do
     #with bundled JDKs
     @bundles_jdk = true
     license_details = ['APACHE-LICENSE-2.0', "-oss", oss_exclude_paths]
@@ -224,7 +224,7 @@ namespace "artifact" do
   end
 
   desc "Build an RPM of logstash with all dependencies"
-  task "rpm_oss" => ["prepare", "generate_build_metadata"] do
+  task "rpm_oss" => ["prepare-oss", "generate_build_metadata"] do
     #with bundled JDKs
     @bundles_jdk = true
     puts("[artifact:rpm] building rpm OSS package x86_64")
@@ -256,7 +256,7 @@ namespace "artifact" do
   end
 
   desc "Build a DEB of logstash with all dependencies"
-  task "deb_oss" => ["prepare", "generate_build_metadata"] do
+  task "deb_oss" => ["prepare-oss", "generate_build_metadata"] do
     #with bundled JDKs
     @bundles_jdk = true
     puts("[artifact:deb_oss] building deb OSS package x86_64")
@@ -300,7 +300,7 @@ namespace "artifact" do
   end
 
   desc "Build OSS docker image"
-  task "docker_oss" => ["prepare", "generate_build_metadata", "archives_oss"] do
+  task "docker_oss" => ["prepare-oss", "generate_build_metadata", "archives_oss"] do
     puts("[docker_oss] Building OSS docker image")
     build_docker('oss')
   end
@@ -321,7 +321,7 @@ namespace "artifact" do
   end
 
   desc "Generate Dockerfile for oss images"
-  task "dockerfile_oss" => ["prepare", "generate_build_metadata"] do
+  task "dockerfile_oss" => ["prepare-oss", "generate_build_metadata"] do
     puts("[dockerfiles] Building oss Dockerfile")
     build_dockerfile('oss')
   end
@@ -348,17 +348,19 @@ namespace "artifact" do
   task "build" => [:generate_build_metadata] do
     Rake::Task["artifact:gems"].invoke unless SNAPSHOT_BUILD
     Rake::Task["artifact:deb"].invoke
-    Rake::Task["artifact:deb_oss"].invoke
     Rake::Task["artifact:rpm"].invoke
-    Rake::Task["artifact:rpm_oss"].invoke
     Rake::Task["artifact:archives"].invoke
-    Rake::Task["artifact:archives_oss"].invoke
+
     unless ENV['SKIP_DOCKER'] == "1"
       Rake::Task["artifact:docker"].invoke
-      Rake::Task["artifact:docker_oss"].invoke
       Rake::Task["artifact:docker_ubi8"].invoke
       Rake::Task["artifact:dockerfiles"].invoke
+      Rake::Task["artifact:docker_oss"].invoke
     end
+
+    Rake::Task["artifact:deb_oss"].invoke
+    Rake::Task["artifact:rpm_oss"].invoke
+    Rake::Task["artifact:archives_oss"].invoke
   end
 
   task "build_docker_full" => [:generate_build_metadata] do
@@ -378,6 +380,7 @@ namespace "artifact" do
 
   task "generate_build_metadata" do
     require 'time'
+    require 'tempfile'
 
     return if defined?(BUILD_METADATA_FILE)
     BUILD_METADATA_FILE = Tempfile.new('build.rb')
@@ -446,6 +449,12 @@ namespace "artifact" do
     end
   end
 
+  task "prepare-oss" do
+    if ENV['SKIP_PREPARE'] != "1"
+      %w[bootstrap plugin:install-default plugin:remove-non-oss-plugins artifact:clean-bundle-config].each {|task| Rake::Task[task].invoke }
+    end
+  end
+
   def ensure_logstash_version_constant_defined
     # we do not want this file required when rake (ruby) parses this file
     # only when there is a task executing, not at the very top of this file

rakelib/default_plugins.rb

@@ -51,5 +51,8 @@ def self.fetch_plugins_for(type)
     CORE_SPECS_PLUGINS = self.fetch_plugins_for("core-specs").freeze
 
     ALL_PLUGINS_SKIP_LIST = Regexp.union(self.fetch_plugins_for("skip-list")).freeze
+
+    # default plugins will be installed and we exclude only installed plugins from OSS
+    OSS_EXCLUDED_PLUGINS = DEFAULT_PLUGINS & self.fetch_plugins_for("skip-oss")
   end
 end

rakelib/plugin.rake

@@ -24,6 +24,11 @@ namespace "plugin" do
     LogStash::PluginManager::Main.run("bin/logstash-plugin", ["install"] + args)
   end
 
+  def remove_plugin(plugin)
+    require_relative "../lib/pluginmanager/main"
+    LogStash::PluginManager::Main.run("bin/logstash-plugin", ["remove", plugin])
+  end
+
   task "install-base" => "bootstrap" do
     puts("[plugin:install-base] Installing base dependencies")
     install_plugins("--development",  "--preserve")
@@ -61,6 +66,18 @@ namespace "plugin" do
     task.reenable # Allow this task to be run again
   end
 
+  task "remove-non-oss-plugins" do |task, _|
+    puts("[plugin:remove-non-oss-plugins] Removing non-OSS plugins")
+
+    LogStash::RakeLib::OSS_EXCLUDED_PLUGINS.each do |plugin|
+      remove_plugin(plugin)
+      # gem folder and spec file still stay after removing the plugin
+      FileUtils.rm_r(Dir.glob("#{LogStash::Environment::BUNDLE_DIR}/**/gems/#{plugin}*"))
+      FileUtils.rm_r(Dir.glob("#{LogStash::Environment::BUNDLE_DIR}/**/specifications/#{plugin}*.gemspec"))
+    end
+    task.reenable # Allow this task to be run again
+  end
+
   task "clean-local-core-gem", [:name, :path] do |task, args|
     name = args[:name]
     path = args[:path]

rakelib/plugins-metadata.json

@@ -120,8 +120,9 @@
     "skip-list": false
   },
   "logstash-filter-elastic_integration": {
-    "default-plugins": false,
-    "skip-list": false
+    "default-plugins": true,
+    "skip-list": false,
+    "skip-oss": true
   },
   "logstash-filter-elasticsearch": {
     "default-plugins": true,