(puppetlabsGH-1934) Add support for selective sync of plugins to target nodes

* **Added support for selective sync of plugins to target** ([puppetlabs#1934](puppetlabs#1934))

!feature
This feature allows a plan author to specificy what plugins need to be synced to the target node. You can do this by add an array of module names to the options parameter. Here is an example:

    apply($target, ‘required_modules => [‘stdlib’] ) {
      notice 'Hallo'
    }

The apply block here ony sync’s the stdlib module to the target. The same feature also
works on apply_prep functions. Here is an example:

    apply_prep($target, ‘required_modules => [‘stdlib’] )

When no required modules are specified, *ALL* plugins from *ALL* modules are synced.

Author: hajee

bolt-modules/boltlib/lib/puppet/functions/apply_prep.rb

@@ -13,10 +13,13 @@
 # > **Note:** Not available in apply block
 Puppet::Functions.create_function(:apply_prep) do
   # @param targets A pattern or array of patterns identifying a set of targets.
+  # @param options Options hash.
+  # @option options [Array] _required_modules An array of modules to sync to the target.
   # @example Prepare targets by name.
   #   apply_prep('target1,target2')
   dispatch :apply_prep do
     param 'Boltlib::TargetSpec', :targets
+    optional_param 'Hash[String, Data]', :options
   end
 
   def script_compiler
@@ -60,18 +63,34 @@ def executor
     @executor ||= Puppet.lookup(:bolt_executor)
   end
 
-  def apply_prep(target_spec)
+  def apply_prep(target_spec, options = {})
     unless Puppet[:tasks]
       raise Puppet::ParseErrorWithIssue
         .from_issue_and_stack(Bolt::PAL::Issues::PLAN_OPERATION_NOT_SUPPORTED_WHEN_COMPILING, action: 'apply_prep')
     end
 
+    options = options.transform_keys { |k| k.sub(/^_/, '').to_sym }
+
     applicator = Puppet.lookup(:apply_executor)
 
     executor.report_function_call(self.class.name)
 
     targets = inventory.get_targets(target_spec)
 
+    required_modules = options[:required_modules].nil? ? nil : Array(options[:required_modules])
+    if required_modules&.any?
+      Puppet.debug("Syncing only required modules: #{required_modules.join(',')}.")
+    end
+
+    # Gather facts, including custom facts
+    plugins = applicator.build_plugin_tarball do |mod|
+      next unless required_modules.nil? || required_modules.include?(mod.name)
+      search_dirs = []
+      search_dirs << mod.plugins if mod.plugins?
+      search_dirs << mod.pluginfacts if mod.pluginfacts?
+      search_dirs
+    end
+
     executor.log_action('install puppet and gather facts', targets) do
       executor.without_default_logging do
         # Skip targets that include the puppet-agent feature, as we know an agent will be available.
@@ -109,14 +128,6 @@ def apply_prep(target_spec)
           need_install_targets.each { |target| set_agent_feature(target) }
         end
 
-        # Gather facts, including custom facts
-        plugins = applicator.build_plugin_tarball do |mod|
-          search_dirs = []
-          search_dirs << mod.plugins if mod.plugins?
-          search_dirs << mod.pluginfacts if mod.pluginfacts?
-          search_dirs
-        end
-
         task = applicator.custom_facts_task
         arguments = { 'plugins' => Puppet::Pops::Types::PSensitiveType::Sensitive.new(plugins) }
         results = executor.run_task(targets, task, arguments)

bolt-modules/boltlib/spec/functions/apply_prep_spec.rb

@@ -217,6 +217,28 @@
     end
   end
 
+  context 'with required_modules specified' do
+    let(:hostnames)         { %w[foo bar] }
+    let(:targets)           { hostnames.map { |h| inventory.get_target(h) } }
+    let(:fact)              { { 'osfamily' => 'none' } }
+    let(:custom_facts_task) { Bolt::Task.new('custom_facts_task') }
+
+    before(:each) do
+      applicator.stubs(:build_plugin_tarball).returns(:tarball)
+      applicator.stubs(:custom_facts_task).returns(custom_facts_task)
+      targets.each { |target| inventory.set_feature(target, 'puppet-agent') }
+    end
+
+    it 'only uses required plugins' do
+      facts = Bolt::ResultSet.new(targets.map { |t| Bolt::Result.new(t, value: fact) })
+      executor.expects(:run_task).with(targets, custom_facts_task, includes('plugins')).returns(facts)
+
+      Puppet.expects(:debug).at_least(1)
+      Puppet.expects(:debug).with("Syncing only required modules: non-existing-module.")
+      is_expected.to run.with_params(hostnames.join(','), 'required_modules' => ['non-existing-module']).and_return(nil)
+    end
+  end
+
   context 'without tasks enabled' do
     let(:tasks_enabled) { false }
     it 'fails and reports that apply_prep is not available' do

lib/bolt/applicator.rb

@@ -18,7 +18,6 @@ def initialize(inventory, executor, modulepath, plugin_dirs, project,
                    pdb_client, hiera_config, max_compiles, apply_settings)
       # lazy-load expensive gem code
       require 'concurrent'
-
       @inventory = inventory
       @executor = executor
       @modulepath = modulepath || []
@@ -30,17 +29,6 @@ def initialize(inventory, executor, modulepath, plugin_dirs, project,
 
       @pool = Concurrent::ThreadPoolExecutor.new(max_threads: max_compiles)
       @logger = Logging.logger[self]
-      @plugin_tarball = Concurrent::Delay.new do
-        build_plugin_tarball do |mod|
-          search_dirs = []
-          search_dirs << mod.plugins if mod.plugins?
-          search_dirs << mod.pluginfacts if mod.pluginfacts?
-          search_dirs << mod.files if mod.files?
-          type_files = "#{mod.path}/types"
-          search_dirs << type_files if File.exist?(type_files)
-          search_dirs
-        end
-      end
     end
 
     private def libexec
@@ -188,7 +176,6 @@ def count_statements(ast)
 
     def apply_ast(raw_ast, targets, options, plan_vars = {})
       ast = Puppet::Pops::Serialization::ToDataConverter.convert(raw_ast, rich_data: true, symbol_to_string: true)
-
       # Serialize as pcore for *Result* objects
       plan_vars = Puppet::Pops::Serialization::ToDataConverter.convert(plan_vars,
                                                                        rich_data: true,
@@ -206,9 +193,26 @@ def apply_ast(raw_ast, targets, options, plan_vars = {})
         # This data isn't available on the target config hash
         config: @inventory.transport_data_get
       }
-
       description = options[:description] || 'apply catalog'
 
+      required_modules = options[:required_modules].nil? ? nil : Array(options[:required_modules])
+      if required_modules&.any?
+        @logger.debug("Syncing only required modules: #{required_modules.join(',')}.")
+      end
+
+      @plugin_tarball = Concurrent::Delay.new do
+        build_plugin_tarball do |mod|
+          next unless required_modules.nil? || required_modules.include?(mod.name)
+          search_dirs = []
+          search_dirs << mod.plugins if mod.plugins?
+          search_dirs << mod.pluginfacts if mod.pluginfacts?
+          search_dirs << mod.files if mod.files?
+          type_files = "#{mod.path}/types"
+          search_dirs << type_files if File.exist?(type_files)
+          search_dirs
+        end
+      end
+
       r = @executor.log_action(description, targets) do
         futures = targets.map do |target|
           Concurrent::Future.execute(executor: @pool) do
@@ -235,6 +239,7 @@ def apply_ast(raw_ast, targets, options, plan_vars = {})
                   result
                 end
               else
+
                 arguments = {
                   'catalog' => Puppet::Pops::Types::PSensitiveType::Sensitive.new(catalog),
                   'plugins' => Puppet::Pops::Types::PSensitiveType::Sensitive.new(plugins),

spec/bolt/applicator_spec.rb

@@ -10,6 +10,7 @@
 
 describe Bolt::Applicator do
   let(:uri) { 'foobar' }
+  let(:plugindirs) { [] }
   let(:target) { inventory.get_target(uri) }
   let(:inventory) { Bolt::Inventory.empty }
   let(:executor) { Bolt::Executor.new }
@@ -20,7 +21,7 @@
   end
   let(:pdb_client) { Bolt::PuppetDB::Client.new(config) }
   let(:modulepath) { [Bolt::PAL::BOLTLIB_PATH, Bolt::PAL::MODULES_PATH] }
-  let(:applicator) { Bolt::Applicator.new(inventory, executor, modulepath, [], nil, pdb_client, nil, 2, {}) }
+  let(:applicator) { Bolt::Applicator.new(inventory, executor, modulepath, plugindirs, nil, pdb_client, nil, 2, {}) }
   let(:ast) { { 'resources' => [] } }
 
   let(:report) {
@@ -115,6 +116,61 @@
 
     let(:scope) { double('scope') }
 
+    context 'without required modules specified (default)' do
+      before do
+        allow(Logging).to receive(:logger).and_return(mock_logger)
+        allow(mock_logger).to receive(:[]).and_return(mock_logger)
+        allow(mock_logger).to receive(:'level=').with(any_args)
+        allow(mock_logger).to receive(:debug).with(any_args)
+      end
+
+      let(:mock_logger) { instance_double("Logging.logger") }
+      let(:plugindirs) { modulepath }
+
+      it 'syncs all modules' do
+        #
+        # Use a variable here instead of the Rspec let, so we can mock the logger
+        #
+        applicator = Bolt::Applicator.new(inventory, executor, modulepath, plugindirs, nil, pdb_client, nil, 2, {})
+        expect(applicator).to receive(:compile).and_return(ast)
+        result = Bolt::Result.new(target, value: report)
+        allow_any_instance_of(Bolt::Transport::SSH).to receive(:batch_task).and_return(result)
+        allow(Bolt::ApplyResult).to receive(:puppet_missing_error).with(result).and_return(nil)
+
+        expect(mock_logger).to receive(:debug).with(/Packing plugin/).at_least(:once)
+        expect(mock_logger).to_not receive(:debug).with(/Syncing only required modules/)
+        applicator.apply([target], :body, scope)
+      end
+    end
+
+    context 'required modules specified' do
+      before do
+        allow(Logging).to receive(:logger).and_return(mock_logger)
+        allow(mock_logger).to receive(:[]).and_return(mock_logger)
+        allow(mock_logger).to receive(:'level=').with(any_args)
+        allow(mock_logger).to receive(:debug).with(any_args)
+      end
+
+      let(:mock_logger) { instance_double("Logging.logger") }
+      let(:plugindirs) { modulepath }
+
+      it 'syncs only required modules' do
+        #
+        # Use a variable here instead of the Rspec let, so we can mock the logger
+        #
+        applicator = Bolt::Applicator.new(inventory, executor, modulepath, plugindirs, nil, pdb_client, nil, 2, {})
+        expect(applicator).to receive(:compile).and_return(ast)
+        result = Bolt::Result.new(target, value: report)
+        allow_any_instance_of(Bolt::Transport::SSH).to receive(:batch_task).and_return(result)
+        allow(Bolt::ApplyResult).to receive(:puppet_missing_error).with(result).and_return(nil)
+
+        expect(mock_logger).to_not receive(:debug).with(/Packing plugin/)
+        expect(mock_logger).to receive(:debug).with('Syncing only required modules: just_a_module_name.')
+
+        applicator.apply_ast(:body, [target], { required_modules: ['just_a_module_name'] })
+      end
+    end
+
     it 'replaces failures to find Puppet' do
       expect(applicator).to receive(:compile).and_return(ast)
       result = Bolt::Result.new(target, value: report)