Merge pull request #1935 from enterprisemodules/selective_module_sync

(GH-1934) Add support for selective sync of plugins to target nodes

bolt-modules/boltlib/lib/puppet/functions/apply_prep.rb

@@ -13,10 +13,13 @@
 # > **Note:** Not available in apply block
 Puppet::Functions.create_function(:apply_prep) do
   # @param targets A pattern or array of patterns identifying a set of targets.
+  # @param options Options hash.
+  # @option options [Array] _required_modules An array of modules to sync to the target.
   # @example Prepare targets by name.
   #   apply_prep('target1,target2')
   dispatch :apply_prep do
     param 'Boltlib::TargetSpec', :targets
+    optional_param 'Hash[String, Data]', :options
   end
 
   def script_compiler
@@ -60,18 +63,34 @@ def executor
     @executor ||= Puppet.lookup(:bolt_executor)
   end
 
-  def apply_prep(target_spec)
+  def apply_prep(target_spec, options = {})
     unless Puppet[:tasks]
       raise Puppet::ParseErrorWithIssue
         .from_issue_and_stack(Bolt::PAL::Issues::PLAN_OPERATION_NOT_SUPPORTED_WHEN_COMPILING, action: 'apply_prep')
     end
 
+    options = options.transform_keys { |k| k.sub(/^_/, '').to_sym }
+
     applicator = Puppet.lookup(:apply_executor)
 
     executor.report_function_call(self.class.name)
 
     targets = inventory.get_targets(target_spec)
 
+    required_modules = options[:required_modules].nil? ? nil : Array(options[:required_modules])
+    if required_modules&.any?
+      Puppet.debug("Syncing only required modules: #{required_modules.join(',')}.")
+    end
+
+    # Gather facts, including custom facts
+    plugins = applicator.build_plugin_tarball do |mod|
+      next unless required_modules.nil? || required_modules.include?(mod.name)
+      search_dirs = []
+      search_dirs << mod.plugins if mod.plugins?
+      search_dirs << mod.pluginfacts if mod.pluginfacts?
+      search_dirs
+    end
+
     executor.log_action('install puppet and gather facts', targets) do
       executor.without_default_logging do
         # Skip targets that include the puppet-agent feature, as we know an agent will be available.
@@ -109,14 +128,6 @@ def apply_prep(target_spec)
           need_install_targets.each { |target| set_agent_feature(target) }
         end
 
-        # Gather facts, including custom facts
-        plugins = applicator.build_plugin_tarball do |mod|
-          search_dirs = []
-          search_dirs << mod.plugins if mod.plugins?
-          search_dirs << mod.pluginfacts if mod.pluginfacts?
-          search_dirs
-        end
-
         task = applicator.custom_facts_task
         arguments = { 'plugins' => Puppet::Pops::Types::PSensitiveType::Sensitive.new(plugins) }
         results = executor.run_task(targets, task, arguments)

bolt-modules/boltlib/spec/functions/apply_prep_spec.rb

@@ -217,6 +217,28 @@
     end
   end
 
+  context 'with required_modules specified' do
+    let(:hostnames)         { %w[foo bar] }
+    let(:targets)           { hostnames.map { |h| inventory.get_target(h) } }
+    let(:fact)              { { 'osfamily' => 'none' } }
+    let(:custom_facts_task) { Bolt::Task.new('custom_facts_task') }
+
+    before(:each) do
+      applicator.stubs(:build_plugin_tarball).returns(:tarball)
+      applicator.stubs(:custom_facts_task).returns(custom_facts_task)
+      targets.each { |target| inventory.set_feature(target, 'puppet-agent') }
+    end
+
+    it 'only uses required plugins' do
+      facts = Bolt::ResultSet.new(targets.map { |t| Bolt::Result.new(t, value: fact) })
+      executor.expects(:run_task).with(targets, custom_facts_task, includes('plugins')).returns(facts)
+
+      Puppet.expects(:debug).at_least(1)
+      Puppet.expects(:debug).with("Syncing only required modules: non-existing-module.")
+      is_expected.to run.with_params(hostnames.join(','), 'required_modules' => ['non-existing-module']).and_return(nil)
+    end
+  end
+
   context 'without tasks enabled' do
     let(:tasks_enabled) { false }
     it 'fails and reports that apply_prep is not available' do

lib/bolt/applicator.rb

@@ -18,7 +18,6 @@ def initialize(inventory, executor, modulepath, plugin_dirs, project,
                    pdb_client, hiera_config, max_compiles, apply_settings)
       # lazy-load expensive gem code
       require 'concurrent'
-
       @inventory = inventory
       @executor = executor
       @modulepath = modulepath || []
@@ -30,17 +29,6 @@ def initialize(inventory, executor, modulepath, plugin_dirs, project,
 
       @pool = Concurrent::ThreadPoolExecutor.new(max_threads: max_compiles)
       @logger = Logging.logger[self]
-      @plugin_tarball = Concurrent::Delay.new do
-        build_plugin_tarball do |mod|
-          search_dirs = []
-          search_dirs << mod.plugins if mod.plugins?
-          search_dirs << mod.pluginfacts if mod.pluginfacts?
-          search_dirs << mod.files if mod.files?
-          type_files = "#{mod.path}/types"
-          search_dirs << type_files if File.exist?(type_files)
-          search_dirs
-        end
-      end
     end
 
     private def libexec
@@ -188,7 +176,6 @@ def count_statements(ast)
 
     def apply_ast(raw_ast, targets, options, plan_vars = {})
       ast = Puppet::Pops::Serialization::ToDataConverter.convert(raw_ast, rich_data: true, symbol_to_string: true)
-
       # Serialize as pcore for *Result* objects
       plan_vars = Puppet::Pops::Serialization::ToDataConverter.convert(plan_vars,
                                                                        rich_data: true,
@@ -207,9 +194,26 @@ def apply_ast(raw_ast, targets, options, plan_vars = {})
         # This data isn't available on the target config hash
         config: @inventory.transport_data_get
       }
-
       description = options[:description] || 'apply catalog'
 
+      required_modules = options[:required_modules].nil? ? nil : Array(options[:required_modules])
+      if required_modules&.any?
+        @logger.debug("Syncing only required modules: #{required_modules.join(',')}.")
+      end
+
+      @plugin_tarball = Concurrent::Delay.new do
+        build_plugin_tarball do |mod|
+          next unless required_modules.nil? || required_modules.include?(mod.name)
+          search_dirs = []
+          search_dirs << mod.plugins if mod.plugins?
+          search_dirs << mod.pluginfacts if mod.pluginfacts?
+          search_dirs << mod.files if mod.files?
+          type_files = "#{mod.path}/types"
+          search_dirs << type_files if File.exist?(type_files)
+          search_dirs
+        end
+      end
+
       r = @executor.log_action(description, targets) do
         futures = targets.map do |target|
           Concurrent::Future.execute(executor: @pool) do
@@ -236,6 +240,7 @@ def apply_ast(raw_ast, targets, options, plan_vars = {})
                   result
                 end
               else
+
                 arguments = {
                   'catalog' => Puppet::Pops::Types::PSensitiveType::Sensitive.new(catalog),
                   'plugins' => Puppet::Pops::Types::PSensitiveType::Sensitive.new(plugins),

spec/bolt/applicator_spec.rb

@@ -10,6 +10,7 @@
 
 describe Bolt::Applicator do
   let(:uri) { 'foobar' }
+  let(:plugindirs) { [] }
   let(:target) { inventory.get_target(uri) }
   let(:inventory) { Bolt::Inventory.empty }
   let(:executor) { Bolt::Executor.new }
@@ -20,7 +21,7 @@
   end
   let(:pdb_client) { Bolt::PuppetDB::Client.new(config) }
   let(:modulepath) { [Bolt::PAL::BOLTLIB_PATH, Bolt::PAL::MODULES_PATH] }
-  let(:applicator) { Bolt::Applicator.new(inventory, executor, modulepath, [], nil, pdb_client, nil, 2, {}) }
+  let(:applicator) { Bolt::Applicator.new(inventory, executor, modulepath, plugindirs, nil, pdb_client, nil, 2, {}) }
   let(:ast) { { 'resources' => [] } }
 
   let(:report) {
@@ -115,6 +116,61 @@
 
     let(:scope) { double('scope') }
 
+    context 'without required modules specified (default)' do
+      before do
+        allow(Logging).to receive(:logger).and_return(mock_logger)
+        allow(mock_logger).to receive(:[]).and_return(mock_logger)
+        allow(mock_logger).to receive(:'level=').with(any_args)
+        allow(mock_logger).to receive(:debug).with(any_args)
+      end
+
+      let(:mock_logger) { instance_double("Logging.logger") }
+      let(:plugindirs) { modulepath }
+
+      it 'syncs all modules' do
+        #
+        # Use a variable here instead of the Rspec let, so we can mock the logger
+        #
+        applicator = Bolt::Applicator.new(inventory, executor, modulepath, plugindirs, nil, pdb_client, nil, 2, {})
+        expect(applicator).to receive(:compile).and_return(ast)
+        result = Bolt::Result.new(target, value: report)
+        allow_any_instance_of(Bolt::Transport::SSH).to receive(:batch_task).and_return(result)
+        allow(Bolt::ApplyResult).to receive(:puppet_missing_error).with(result).and_return(nil)
+
+        expect(mock_logger).to receive(:debug).with(/Packing plugin/).at_least(:once)
+        expect(mock_logger).to_not receive(:debug).with(/Syncing only required modules/)
+        applicator.apply([target], :body, scope)
+      end
+    end
+
+    context 'required modules specified' do
+      before do
+        allow(Logging).to receive(:logger).and_return(mock_logger)
+        allow(mock_logger).to receive(:[]).and_return(mock_logger)
+        allow(mock_logger).to receive(:'level=').with(any_args)
+        allow(mock_logger).to receive(:debug).with(any_args)
+      end
+
+      let(:mock_logger) { instance_double("Logging.logger") }
+      let(:plugindirs) { modulepath }
+
+      it 'syncs only required modules' do
+        #
+        # Use a variable here instead of the Rspec let, so we can mock the logger
+        #
+        applicator = Bolt::Applicator.new(inventory, executor, modulepath, plugindirs, nil, pdb_client, nil, 2, {})
+        expect(applicator).to receive(:compile).and_return(ast)
+        result = Bolt::Result.new(target, value: report)
+        allow_any_instance_of(Bolt::Transport::SSH).to receive(:batch_task).and_return(result)
+        allow(Bolt::ApplyResult).to receive(:puppet_missing_error).with(result).and_return(nil)
+
+        expect(mock_logger).to_not receive(:debug).with(/Packing plugin/)
+        expect(mock_logger).to receive(:debug).with('Syncing only required modules: just_a_module_name.')
+
+        applicator.apply_ast(:body, [target], { required_modules: ['just_a_module_name'] })
+      end
+    end
+
     it 'replaces failures to find Puppet' do
       expect(applicator).to receive(:compile).and_return(ast)
       result = Bolt::Result.new(target, value: report)