Merge pull request #15459 from jrafanie/bundler_groups_with_runner_sc…

…ript

Introduce bundler groups

Gemfile

@@ -17,31 +17,14 @@ def manageiq_plugin(plugin_name)
   end
 end
 
-manageiq_plugin "manageiq-automation_engine"
-manageiq_plugin "manageiq-content"
-manageiq_plugin "manageiq-providers-amazon"
-manageiq_plugin "manageiq-providers-ansible_tower"
-manageiq_plugin "manageiq-providers-azure"
-manageiq_plugin "manageiq-providers-foreman"
-manageiq_plugin "manageiq-providers-google"
-manageiq_plugin "manageiq-providers-hawkular"
-manageiq_plugin "manageiq-providers-kubernetes"
-manageiq_plugin "manageiq-providers-lenovo"
-manageiq_plugin "manageiq-providers-nuage"
-manageiq_plugin "manageiq-providers-openshift"
-manageiq_plugin "manageiq-providers-openstack"
-manageiq_plugin "manageiq-providers-ovirt"
-manageiq_plugin "manageiq-providers-scvmm"
-manageiq_plugin "manageiq-providers-vmware"
+manageiq_plugin "manageiq-providers-ansible_tower" # can't move this down yet, because we can't autoload ManageIQ::Providers::AnsibleTower::Shared
 manageiq_plugin "manageiq-schema"
-manageiq_plugin "manageiq-ui-classic"
 
 # Unmodified gems
 gem "activerecord-id_regions",        "~>0.2.0"
 gem "activerecord-session_store",     "~>1.0.0"
 gem "acts_as_tree",                   "~>2.1.0" # acts_as_tree needs to be required so that it loads before ancestry
 gem "ancestry",                       "~>2.2.1",       :require => false
-gem "ansible_tower_client",           "~>0.12.2",      :require => false
 gem "bundler",                        ">=1.11.1",      :require => false
 gem "color",                          "~>1.8"
 gem "config",                         "~>1.3.0",       :require => false
@@ -52,42 +35,26 @@ gem "fast_gettext",                   "~>1.2.0"
 gem "gettext_i18n_rails",             "~>1.7.2"
 gem "gettext_i18n_rails_js",          "~>1.1.0"
 gem "hamlit",                         "~>2.7.0"
-gem "htauth",                         "2.0.0",         :require => false
 gem "inifile",                        "~>3.0",         :require => false
-gem "jbuilder",                       "~>2.5.0" # For the REST API
 gem "manageiq-api-client",            "~>0.1.0",       :require => false
 gem "manageiq-network_discovery",     "~>0.1.1",       :require => false
-gem "manageiq-smartstate",            "~>0.1.1",       :require => false
 gem "mime-types",                     "~>2.6.1",       :path => "mime-types-redirector"
 gem "more_core_extensions",           "~>3.2"
 gem "nakayoshi_fork",                 "~>0.0.3"  # provides a more CoW friendly fork (GC a few times before fork)
 gem "net-ldap",                       "~>0.14.0",      :require => false
 gem "net-ping",                       "~>1.7.4",       :require => false
 gem "net-ssh",                        "=3.2.0",        :require => false
-gem "open4",                          "~>1.3.0",       :require => false
 gem "openscap",                       "~>0.4.3",       :require => false
-gem "ovirt-engine-sdk",               "~>4.1.4",       :require => false # Required by the oVirt provider
-gem "ovirt_metrics",                  "~>1.4.1",       :require => false
-gem "pg-pglogical",                   "~>1.1.0",       :require => false
-gem "puma",                           "~>3.3.0"
 gem "query_relation",                 "~>0.1.0",       :require => false
 gem "rails",                          "~>5.0.2"
 gem "rails-i18n",                     "~>5.x"
-gem "responders",                     "~>2.0"
 gem "rest-client",                    "~>2.0.0",       :require => false
 gem "ripper_ruby_parser",                              :require => false
-gem "ruby-dbus" # For external auth
 gem "ruby-progressbar",               "~>1.7.0",       :require => false
 gem "rubyzip",                        "~>1.2.1",       :require => false
-gem "rufus-scheduler",                "~>3.1.3",       :require => false
 gem "rugged",                         "~>0.25.0",      :require => false
-gem "secure_headers",                 "~>3.0.0"
 gem "simple-rss",                     "~>1.3.1",       :require => false
 gem "snmp",                           "~>1.2.0",       :require => false
-gem "sshkey",                         "~>1.8.0",       :require => false
-gem "thin",                           "~>1.7.0",       :require => false
-gem "vmware_web_service",             "~>0.1.1",       :require => false
-gem "websocket-driver",               "~>0.6.3"
 
 # Modified gems (forked on Github)
 gem "amazon_ssa_support",                          :require => false, :git => "https://github.com/ManageIQ/amazon_ssa_support.git", :branch => "master" # Temporary dependency to be moved to manageiq-providers-amazon when officially release
@@ -99,12 +66,126 @@ gem "ruport",                         "=1.7.0",                       :git => "h
 # https://github.com/jeremyevans/ruby-american_date
 gem "american_date"
 
-group :ui_dependencies do # Added to Bundler.require in config/application.rb
+# Make sure to tag your new bundler group with the manageiq_default group in addition to your specific bundler group name.
+# This default is used to automatically require all of our gems in processes that don't specify which bundler groups they want.
+#
+### providers
+group :amazon, :manageiq_default do
+  manageiq_plugin "manageiq-providers-amazon"
+end
+
+group :ansible, :manageiq_default do
+  gem "ansible_tower_client",           "~>0.12.2",      :require => false
+end
+
+group :azure, :manageiq_default do
+  manageiq_plugin "manageiq-providers-azure"
+end
+
+group :foreman, :manageiq_default do
+  manageiq_plugin "manageiq-providers-foreman"
+  gem "foreman_api_client",             ">=0.1.0",   :require => false, :git => "https://github.com/ManageIQ/foreman_api_client.git", :branch => "master"
+end
+
+group :google, :manageiq_default do
+  manageiq_plugin "manageiq-providers-google"
+  gem "fog-google",                     ">=0.5.2",       :require => false
+  gem "google-api-client",              "~>0.8.6",       :require => false
+end
+
+group :hawkular, :manageiq_default do
+  manageiq_plugin "manageiq-providers-hawkular"
+end
+
+group :kubernetes, :openshift, :manageiq_default do
+  manageiq_plugin "manageiq-providers-kubernetes"
+end
+
+group :lenovo, :manageiq_default do
+  manageiq_plugin "manageiq-providers-lenovo"
+end
+
+group :nuage, :manageiq_default do
+  manageiq_plugin "manageiq-providers-nuage"
+end
+
+group :openshift, :manageiq_default do
+  manageiq_plugin "manageiq-providers-openshift"
+  gem "htauth",                         "2.0.0",         :require => false # used by container deployment
+end
+
+group :openstack, :manageiq_default do
+  manageiq_plugin "manageiq-providers-openstack"
+end
+
+group :ovirt, :manageiq_default do
+  manageiq_plugin "manageiq-providers-ovirt"
+  gem "ovirt-engine-sdk",               "~>4.1.4",       :require => false # Required by the oVirt provider
+  gem "ovirt_metrics",                  "~>1.4.1",       :require => false
+end
+
+group :scvmm, :manageiq_default do
+  manageiq_plugin "manageiq-providers-scvmm"
+end
+
+group :vmware, :manageiq_default do
+  manageiq_plugin "manageiq-providers-vmware"
+  gem "vmware_web_service",             "~>0.1.1",       :require => false
+end
+
+### shared dependencies
+group :google, :openshift, :manageiq_default do
+  gem "sshkey",                         "~>1.8.0",       :require => false
+end
+
+group :automate, :cockpit, :manageiq_default do
+  gem "open4",                          "~>1.3.0",       :require => false
+end
+
+### end of provider bundler groups
+
+group :automate, :seed, :manageiq_default do
+  manageiq_plugin "manageiq-automation_engine"
+end
+
+group :replication, :manageiq_default do
+  gem "pg-pglogical",                   "~>1.1.0",       :require => false
+end
+
+group :rest_api, :manageiq_default do
+  gem "jbuilder",                       "~>2.5.0" # For the REST API
+end
+
+group :scheduler, :manageiq_default do
+  gem "rufus-scheduler",                "~>3.1.3",       :require => false
+end
+
+group :seed, :manageiq_default do
+  manageiq_plugin "manageiq-content"
+end
+
+group :smartstate, :manageiq_default do
+  gem "manageiq-smartstate",            "~>0.1.1",       :require => false
+end
+
+group :ui_dependencies, :manageiq_default do # Added to Bundler.require in config/application.rb
+  manageiq_plugin "manageiq-ui-classic"
   # Modified gems (forked on Github)
   gem "font-fabulous",                                                :git => "https://github.com/ManageIQ/font-fabulous.git", :branch => "master" # FIXME: this is just a temporary solution and it'll go to the ui-classic later
   gem "jquery-rjs",                   "=0.1.1",                       :git => "https://github.com/ManageIQ/jquery-rjs.git", :tag => "v0.1.1-1"
 end
 
+group :web_server, :manageiq_default do
+  gem "puma",                           "~>3.3.0"
+  gem "responders",                     "~>2.0"
+  gem "ruby-dbus" # For external auth
+  gem "secure_headers",                 "~>3.0.0"
+end
+
+group :web_socket, :manageiq_default do
+  gem "websocket-driver",               "~>0.6.3"
+end
+
 ### Start of gems excluded from the appliances.
 # The gems listed below do not need to be packaged until we find it necessary or useful.
 # Only add gems here that we do not need on an appliance.

app/models/application_record.rb

@@ -15,6 +15,8 @@ class ApplicationRecord < ActiveRecord::Base
   extend ArTableLock
 
   # FIXME: UI code - decorator support
-  extend MiqDecorator::Klass
-  include MiqDecorator::Instance
+  if defined?(ManageIQ::UI::Classic::Engine)
+    extend MiqDecorator::Klass
+    include MiqDecorator::Instance
+  end
 end

app/models/miq_server/worker_management/monitor/class_names.rb

@@ -5,7 +5,7 @@ module MiqServer::WorkerManagement::Monitor::ClassNames
 
   # This are loaded from `lib/workers/miq_worker_types.rb`, and are just a
   # memory reference to them for compatability.
-  MONITOR_CLASS_NAMES               = MIQ_WORKER_TYPES
+  MONITOR_CLASS_NAMES               = MIQ_WORKER_TYPES.keys
   MONITOR_CLASS_NAMES_IN_KILL_ORDER = MIQ_WORKER_TYPES_IN_KILL_ORDER
 
   module ClassMethods

app/models/miq_worker.rb

@@ -370,7 +370,7 @@ def self.runner_script
   end
 
   def start_runner_via_spawn
-    pid = Kernel.spawn(self.class.build_command_line(guid), :out => "/dev/null", :err => [Rails.root.join("log", "evm.log"), "a"])
+    pid = Kernel.spawn(self.class.build_command_line(guid), [:out, :err] => [Rails.root.join("log", "evm.log"), "a"])
     Process.detach(pid)
     pid
   end

app/models/mixins/miq_web_server_runner_mixin.rb

@@ -11,7 +11,7 @@ def do_work
   def do_before_work_loop
     @worker.release_db_connection
 
-    # Since puma/thin traps interrupts, log that we're going away and update our worker row
+    # Since puma traps interrupts, log that we're going away and update our worker row
     at_exit { do_exit("Exit request received.") }
   end
 

app/models/mixins/miq_web_server_worker_mixin.rb

@@ -24,9 +24,6 @@ def preload_for_console
     end
 
     def preload_for_worker_role
-      # Make these constants globally available
-      ::UiConstants
-
       configure_secret_token
     end
 

config/application.rb

@@ -9,8 +9,26 @@
 require 'sprockets/railtie'
 require 'action_cable/engine'
 
+# We use bundler groups to select which dependencies to require in our different processes.
+#  * Anything not in a group are in bundler's 'default' group, and are required all the time
+#  * Anything in development, test, or production will get required by Bundler.require(*Rails.groups) in application.rb
+#    See: https://github.com/rails/rails/blob/c48b21685f4fec1c7a1c9b4e0dde4da89140ee22/railties/lib/rails.rb#L81-L101
+#
+#  Loading application.rb requires any additional BUNDLER_GROUPS based on the environment variable.
+#  This variable should be a comma separated list of groups.
+#  The default BUNDLER_GROUPS below includes all bundler groups not in the Rails.groups.
+#
+ENV['BUNDLER_GROUPS'] ||= "manageiq_default"
+
 if defined?(Bundler)
-  Bundler.require(*Rails.groups, :ui_dependencies)
+  groups = ENV['BUNDLER_GROUPS'].split(",").collect(&:to_sym)
+
+  if $DEBUG
+    puts "** Loading Rails bundler groups: #{Rails.groups.inspect}"
+    puts "** Loading other bundler groups: #{groups.inspect}"
+  end
+
+  Bundler.require(*Rails.groups, *groups)
 end
 
 module Vmdb

config/initializers/secure_headers.rb

@@ -1,15 +1,17 @@
-SecureHeaders::Configuration.default do |config|
-  config.hsts = "max-age=#{20.years.to_i}"
-  config.x_frame_options = 'SAMEORIGIN'
-  config.x_content_type_options = "nosniff"
-  config.x_xss_protection = "1; mode=block"
-  config.csp = {
-    :report_only => false,
-    :default_src => ["'self'"],
-    :frame_src   => ["'self'"],
-    :connect_src => ["'self'"],
-    :style_src   => ["'unsafe-inline'", "'self'"],
-    :script_src  => ["'unsafe-eval'", "'unsafe-inline'", "'self'"],
-    :report_uri  => ["/dashboard/csp_report"]
-  }
+if defined?(SecureHeaders)
+  SecureHeaders::Configuration.default do |config|
+    config.hsts = "max-age=#{20.years.to_i}"
+    config.x_frame_options = 'SAMEORIGIN'
+    config.x_content_type_options = "nosniff"
+    config.x_xss_protection = "1; mode=block"
+    config.csp = {
+      :report_only => false,
+      :default_src => ["'self'"],
+      :frame_src   => ["'self'"],
+      :connect_src => ["'self'"],
+      :style_src   => ["'unsafe-inline'", "'self'"],
+      :script_src  => ["'unsafe-eval'", "'unsafe-inline'", "'self'"],
+      :report_uri  => ["/dashboard/csp_report"]
+    }
+  end
 end

config/puma.rb

@@ -1,7 +1,7 @@
 # We run puma through rails server with options here:
 # https://github.com/ManageIQ/manageiq/blob/2920f76fe609147a6e7c68245a3717d55a7ece7e/app/models/mixins/miq_web_server_worker_mixin.rb#L162-L175
 #
-# Note, Rails::Server and Rack::Server don't expose all puma/thin web server options.
+# Note, Rails::Server and Rack::Server don't expose all puma web server options.
 #
 # As of puma 3.0.0+, puma specific options can be put in config/puma.rb, see:
 # https://github.com/puma/puma/tree/a3136985887d44c79e623b1408a41779b71d8b23#configuration

lib/workers/bin/run_single_worker.rb

@@ -38,18 +38,19 @@
 require File.expand_path("../miq_worker_types", __dir__)
 
 if options[:list]
-  puts ::MIQ_WORKER_TYPES
+  puts ::MIQ_WORKER_TYPES.keys
   exit
 end
 opt_parser.abort(opt_parser.help) unless worker_class
 
-unless ::MIQ_WORKER_TYPES.include?(worker_class)
+unless ::MIQ_WORKER_TYPES.keys.include?(worker_class)
   puts "ERR:  `#{worker_class}` WORKER CLASS NOT FOUND!  Please run with `-l` to see possible worker class names."
   exit 1
 end
 
 # Skip heartbeating with single worker
 ENV["DISABLE_MIQ_WORKER_HEARTBEAT"] ||= options[:heartbeat] ? nil : '1'
+ENV["BUNDLER_GROUPS"] = MIQ_WORKER_TYPES[worker_class].join(',')
 
 require File.expand_path("../../../config/environment", __dir__)