(puppetlabsGH-1934) Add support for selective sync of plugins to targ…

…et nodes

* **Added support for selective sync of plugins to target** ([puppetlabs#1934](puppetlabs#1934))

This feature allows a plan author to specificy what plugins need to be synced to the target node. You can do this by add an array of module names to the options parameter. Here is an example:

    apply($target, ‘required_modules => [‘stdlib’] ) {
      notice 'Hallo'
    }

The apply block here ony sync’s the stdlib module to the target. The same feature also
works on apply_prep functions. Here is an example:

    apply_prep($target, ‘required_modules => [‘stdlib’] )

When no required modules are specified, *ALL* plugins from *ALL* modules are synced.

!feature

bolt-modules/boltlib/lib/puppet/functions/apply_prep.rb

@@ -13,10 +13,12 @@
 # > **Note:** Not available in apply block
 Puppet::Functions.create_function(:apply_prep) do
   # @param targets A pattern or array of patterns identifying a set of targets.
+  # @param options Options hash. For now only supports `required_modules` as an array of modules to sync
   # @example Prepare targets by name.
   #   apply_prep('target1,target2')
   dispatch :apply_prep do
     param 'Boltlib::TargetSpec', :targets
+    optional_param 'Hash[String, Data]', :options
   end
 
   def script_compiler
@@ -60,18 +62,25 @@ def executor
     @executor ||= Puppet.lookup(:bolt_executor)
   end
 
-  def apply_prep(target_spec)
+  def apply_prep(target_spec, options = {})
     unless Puppet[:tasks]
       raise Puppet::ParseErrorWithIssue
         .from_issue_and_stack(Bolt::PAL::Issues::PLAN_OPERATION_NOT_SUPPORTED_WHEN_COMPILING, action: 'apply_prep')
     end
 
+    options.transform_keys { |k| k.sub(/^_/, '').to_sym }
+
     applicator = Puppet.lookup(:apply_executor)
 
     executor.report_function_call(self.class.name)
 
     targets = inventory.get_targets(target_spec)
 
+    @required_modules = Array(options['required_modules'])
+    if @required_modules.any?
+      Puppet.debug("Syncing only required modules: #{@required_modules.join(',')}.")
+    end
+
     executor.log_action('install puppet and gather facts', targets) do
       executor.without_default_logging do
         # Skip targets that include the puppet-agent feature, as we know an agent will be available.
@@ -111,6 +120,7 @@ def apply_prep(target_spec)
 
         # Gather facts, including custom facts
         plugins = applicator.build_plugin_tarball do |mod|
+          next if @required_modules.any? && !@required_modules.include?(mod)
           search_dirs = []
           search_dirs << mod.plugins if mod.plugins?
           search_dirs << mod.pluginfacts if mod.pluginfacts?

bolt-modules/boltlib/spec/functions/apply_prep_spec.rb

@@ -217,6 +217,28 @@
     end
   end
 
+  context 'with required_modules specified' do
+    let(:hostnames)         { %w[foo bar] }
+    let(:targets)           { hostnames.map { |h| inventory.get_target(h) } }
+    let(:fact)              { { 'osfamily' => 'none' } }
+    let(:custom_facts_task) { Bolt::Task.new('custom_facts_task') }
+
+    before(:each) do
+      applicator.stubs(:build_plugin_tarball).returns(:tarball)
+      applicator.stubs(:custom_facts_task).returns(custom_facts_task)
+      targets.each { |target| inventory.set_feature(target, 'puppet-agent') }
+    end
+
+    it 'only uses required plugins' do
+      facts = Bolt::ResultSet.new(targets.map { |t| Bolt::Result.new(t, value: fact) })
+      executor.expects(:run_task).with(targets, custom_facts_task, includes('plugins')).returns(facts)
+
+      Puppet.expects(:debug).at_least(1)
+      Puppet.expects(:debug).with("Syncing only required modules: non-existing-module.")
+      is_expected.to run.with_params(hostnames.join(','), 'required_modules' => ['non-existing-module']).and_return(nil)
+    end
+  end
+
   context 'without tasks enabled' do
     let(:tasks_enabled) { false }
     it 'fails and reports that apply_prep is not available' do

lib/bolt/applicator.rb

@@ -18,7 +18,6 @@ def initialize(inventory, executor, modulepath, plugin_dirs, project,
                    pdb_client, hiera_config, max_compiles, apply_settings)
       # lazy-load expensive gem code
       require 'concurrent'
-
       @inventory = inventory
       @executor = executor
       @modulepath = modulepath || []
@@ -30,17 +29,6 @@ def initialize(inventory, executor, modulepath, plugin_dirs, project,
 
       @pool = Concurrent::ThreadPoolExecutor.new(max_threads: max_compiles)
       @logger = Logging.logger[self]
-      @plugin_tarball = Concurrent::Delay.new do
-        build_plugin_tarball do |mod|
-          search_dirs = []
-          search_dirs << mod.plugins if mod.plugins?
-          search_dirs << mod.pluginfacts if mod.pluginfacts?
-          search_dirs << mod.files if mod.files?
-          type_files = "#{mod.path}/types"
-          search_dirs << type_files if File.exist?(type_files)
-          search_dirs
-        end
-      end
     end
 
     private def libexec
@@ -188,7 +176,6 @@ def count_statements(ast)
 
     def apply_ast(raw_ast, targets, options, plan_vars = {})
       ast = Puppet::Pops::Serialization::ToDataConverter.convert(raw_ast, rich_data: true, symbol_to_string: true)
-
       # Serialize as pcore for *Result* objects
       plan_vars = Puppet::Pops::Serialization::ToDataConverter.convert(plan_vars,
                                                                        rich_data: true,
@@ -206,9 +193,13 @@ def apply_ast(raw_ast, targets, options, plan_vars = {})
         # This data isn't available on the target config hash
         config: @inventory.transport_data_get
       }
-
       description = options[:description] || 'apply catalog'
 
+      @required_modules = Array(options[:required_modules])
+      if @required_modules.any?
+        @logger.debug("Syncing only required modules: #{@required_modules.join(',')}.")
+      end
+
       r = @executor.log_action(description, targets) do
         futures = targets.map do |target|
           Concurrent::Future.execute(executor: @pool) do
@@ -235,9 +226,23 @@ def apply_ast(raw_ast, targets, options, plan_vars = {})
                   result
                 end
               else
+
+                plugin_tarball = Concurrent::Delay.new do
+                  build_plugin_tarball do |mod|
+                    next if @required_modules.any? && !@required_modules.include?(mod.name)
+                    search_dirs = []
+                    search_dirs << mod.plugins if mod.plugins?
+                    search_dirs << mod.pluginfacts if mod.pluginfacts?
+                    search_dirs << mod.files if mod.files?
+                    type_files = "#{mod.path}/types"
+                    search_dirs << type_files if File.exist?(type_files)
+                    search_dirs
+                  end
+                end
+
                 arguments = {
                   'catalog' => Puppet::Pops::Types::PSensitiveType::Sensitive.new(catalog),
-                  'plugins' => Puppet::Pops::Types::PSensitiveType::Sensitive.new(plugins),
+                  'plugins' => Puppet::Pops::Types::PSensitiveType::Sensitive.new(plugins(plugin_tarball)),
                   'apply_settings' => @apply_settings,
                   '_task' => catalog_apply_task.name,
                   '_noop' => options[:noop]
@@ -272,8 +277,8 @@ def apply_ast(raw_ast, targets, options, plan_vars = {})
       r
     end
 
-    def plugins
-      @plugin_tarball.value ||
+    def plugins(for_target_node)
+      for_target_node.value ||
         raise(Bolt::Error.new("Failed to pack module plugins: #{@plugin_tarball.reason}", 'bolt/plugin-error'))
     end
 

spec/bolt/applicator_spec.rb

@@ -10,6 +10,7 @@
 
 describe Bolt::Applicator do
   let(:uri) { 'foobar' }
+  let(:plugindirs) { [] }
   let(:target) { inventory.get_target(uri) }
   let(:inventory) { Bolt::Inventory.empty }
   let(:executor) { Bolt::Executor.new }
@@ -20,7 +21,7 @@
   end
   let(:pdb_client) { Bolt::PuppetDB::Client.new(config) }
   let(:modulepath) { [Bolt::PAL::BOLTLIB_PATH, Bolt::PAL::MODULES_PATH] }
-  let(:applicator) { Bolt::Applicator.new(inventory, executor, modulepath, [], nil, pdb_client, nil, 2, {}) }
+  let(:applicator) { Bolt::Applicator.new(inventory, executor, modulepath, plugindirs, nil, pdb_client, nil, 2, {}) }
   let(:ast) { { 'resources' => [] } }
 
   let(:report) {
@@ -115,6 +116,61 @@
 
     let(:scope) { double('scope') }
 
+    context 'without required modules specified (default)' do
+      before do
+        allow(Logging).to receive(:logger).and_return(mock_logger)
+        allow(mock_logger).to receive(:[]).and_return(mock_logger)
+        allow(mock_logger).to receive(:'level=').with(any_args)
+        allow(mock_logger).to receive(:debug).with(any_args)
+      end
+
+      let(:mock_logger) { instance_double("Logging.logger") }
+      let(:plugindirs) { modulepath }
+
+      it 'syncs all modules' do
+        #
+        # Use a variable here instead of the Rspec let, so we can mock the logger
+        #
+        applicator = Bolt::Applicator.new(inventory, executor, modulepath, plugindirs, nil, pdb_client, nil, 2, {})
+        expect(applicator).to receive(:compile).and_return(ast)
+        result = Bolt::Result.new(target, value: report)
+        allow_any_instance_of(Bolt::Transport::SSH).to receive(:batch_task).and_return(result)
+        allow(Bolt::ApplyResult).to receive(:puppet_missing_error).with(result).and_return(nil)
+
+        expect(mock_logger).to receive(:debug).with(/Packing plugin/).at_least(:once)
+        expect(mock_logger).to_not receive(:debug).with(/Syncing only required modules/)
+        applicator.apply([target], :body, scope)
+      end
+    end
+
+    context 'required modules specified' do
+      before do
+        allow(Logging).to receive(:logger).and_return(mock_logger)
+        allow(mock_logger).to receive(:[]).and_return(mock_logger)
+        allow(mock_logger).to receive(:'level=').with(any_args)
+        allow(mock_logger).to receive(:debug).with(any_args)
+      end
+
+      let(:mock_logger) { instance_double("Logging.logger") }
+      let(:plugindirs) { modulepath }
+
+      it 'syncs only required modules' do
+        #
+        # Use a variable here instead of the Rspec let, so we can mock the logger
+        #
+        applicator = Bolt::Applicator.new(inventory, executor, modulepath, plugindirs, nil, pdb_client, nil, 2, {})
+        expect(applicator).to receive(:compile).and_return(ast)
+        result = Bolt::Result.new(target, value: report)
+        allow_any_instance_of(Bolt::Transport::SSH).to receive(:batch_task).and_return(result)
+        allow(Bolt::ApplyResult).to receive(:puppet_missing_error).with(result).and_return(nil)
+
+        expect(mock_logger).to_not receive(:debug).with(/Packing plugin/)
+        expect(mock_logger).to receive(:debug).with('Syncing only required modules: just_a_module_name.')
+
+        applicator.apply_ast(:body, [target], { required_modules: ['just_a_module_name'] })
+      end
+    end
+
     it 'replaces failures to find Puppet' do
       expect(applicator).to receive(:compile).and_return(ast)
       result = Bolt::Result.new(target, value: report)