From e3eb677f62a148b6bbefec3933da6662f5e9d646 Mon Sep 17 00:00:00 2001 From: Bert Hajee Date: Wed, 24 Jun 2020 09:51:41 +0200 Subject: [PATCH] (GH-1934) Add support for selective sync of plugins to target nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * **Added support for selective sync of plugins to target** ([#1934](https://github.com/puppetlabs/bolt/issues/1934)) This feature allows a plan author to specificy what plugins need to be synced to the target node. You can do this by add an array of module names to the options parameter. Here is an example: apply($target, ‘required_modules => [‘stdlib’] ) { notice 'Hallo' } The apply block here ony sync’s the stdlib module to the target. The same feature also works on apply_prep functions. Here is an example: apply_prep($target, ‘required_modules => [‘stdlib’] ) When no required modules are specified, *ALL* plugins from *ALL* modules are synced. !feature --- .../lib/puppet/functions/apply_prep.rb | 12 +++- .../boltlib/spec/functions/apply_prep_spec.rb | 22 +++++++ lib/bolt/applicator.rb | 39 +++++++------ spec/bolt/applicator_spec.rb | 58 ++++++++++++++++++- 4 files changed, 112 insertions(+), 19 deletions(-) diff --git a/bolt-modules/boltlib/lib/puppet/functions/apply_prep.rb b/bolt-modules/boltlib/lib/puppet/functions/apply_prep.rb index 1e24839fc5..7c2ae5385e 100644 --- a/bolt-modules/boltlib/lib/puppet/functions/apply_prep.rb +++ b/bolt-modules/boltlib/lib/puppet/functions/apply_prep.rb @@ -13,10 +13,12 @@ # > **Note:** Not available in apply block Puppet::Functions.create_function(:apply_prep) do # @param targets A pattern or array of patterns identifying a set of targets. + # @param options Options hash. For now only supports `required_modules` as an array of modules to sync # @example Prepare targets by name. # apply_prep('target1,target2') dispatch :apply_prep do param 'Boltlib::TargetSpec', :targets + optional_param 'Hash[String, Data]', :options end def script_compiler @@ -60,18 +62,25 @@ def executor @executor ||= Puppet.lookup(:bolt_executor) end - def apply_prep(target_spec) + def apply_prep(target_spec, options = {}) unless Puppet[:tasks] raise Puppet::ParseErrorWithIssue .from_issue_and_stack(Bolt::PAL::Issues::PLAN_OPERATION_NOT_SUPPORTED_WHEN_COMPILING, action: 'apply_prep') end + options.transform_keys { |k| k.sub(/^_/, '').to_sym } + applicator = Puppet.lookup(:apply_executor) executor.report_function_call(self.class.name) targets = inventory.get_targets(target_spec) + @required_modules = Array(options['required_modules']) + if @required_modules.any? + Puppet.debug("Syncing only required modules: #{@required_modules.join(',')}.") + end + executor.log_action('install puppet and gather facts', targets) do executor.without_default_logging do # Skip targets that include the puppet-agent feature, as we know an agent will be available. @@ -111,6 +120,7 @@ def apply_prep(target_spec) # Gather facts, including custom facts plugins = applicator.build_plugin_tarball do |mod| + next if @required_modules.any? && !@required_modules.include?(mod) search_dirs = [] search_dirs << mod.plugins if mod.plugins? search_dirs << mod.pluginfacts if mod.pluginfacts? diff --git a/bolt-modules/boltlib/spec/functions/apply_prep_spec.rb b/bolt-modules/boltlib/spec/functions/apply_prep_spec.rb index 0707b11ea8..3f55b01daf 100644 --- a/bolt-modules/boltlib/spec/functions/apply_prep_spec.rb +++ b/bolt-modules/boltlib/spec/functions/apply_prep_spec.rb @@ -217,6 +217,28 @@ end end + context 'with required_modules specified' do + let(:hostnames) { %w[foo bar] } + let(:targets) { hostnames.map { |h| inventory.get_target(h) } } + let(:fact) { { 'osfamily' => 'none' } } + let(:custom_facts_task) { Bolt::Task.new('custom_facts_task') } + + before(:each) do + applicator.stubs(:build_plugin_tarball).returns(:tarball) + applicator.stubs(:custom_facts_task).returns(custom_facts_task) + targets.each { |target| inventory.set_feature(target, 'puppet-agent') } + end + + it 'only uses required plugins' do + facts = Bolt::ResultSet.new(targets.map { |t| Bolt::Result.new(t, value: fact) }) + executor.expects(:run_task).with(targets, custom_facts_task, includes('plugins')).returns(facts) + + Puppet.expects(:debug).at_least(1) + Puppet.expects(:debug).with("Syncing only required modules: non-existing-module.") + is_expected.to run.with_params(hostnames.join(','), 'required_modules' => ['non-existing-module']).and_return(nil) + end + end + context 'without tasks enabled' do let(:tasks_enabled) { false } it 'fails and reports that apply_prep is not available' do diff --git a/lib/bolt/applicator.rb b/lib/bolt/applicator.rb index 2d66598959..87d6d6a0c4 100644 --- a/lib/bolt/applicator.rb +++ b/lib/bolt/applicator.rb @@ -18,7 +18,6 @@ def initialize(inventory, executor, modulepath, plugin_dirs, project, pdb_client, hiera_config, max_compiles, apply_settings) # lazy-load expensive gem code require 'concurrent' - @inventory = inventory @executor = executor @modulepath = modulepath || [] @@ -30,17 +29,6 @@ def initialize(inventory, executor, modulepath, plugin_dirs, project, @pool = Concurrent::ThreadPoolExecutor.new(max_threads: max_compiles) @logger = Logging.logger[self] - @plugin_tarball = Concurrent::Delay.new do - build_plugin_tarball do |mod| - search_dirs = [] - search_dirs << mod.plugins if mod.plugins? - search_dirs << mod.pluginfacts if mod.pluginfacts? - search_dirs << mod.files if mod.files? - type_files = "#{mod.path}/types" - search_dirs << type_files if File.exist?(type_files) - search_dirs - end - end end private def libexec @@ -188,7 +176,6 @@ def count_statements(ast) def apply_ast(raw_ast, targets, options, plan_vars = {}) ast = Puppet::Pops::Serialization::ToDataConverter.convert(raw_ast, rich_data: true, symbol_to_string: true) - # Serialize as pcore for *Result* objects plan_vars = Puppet::Pops::Serialization::ToDataConverter.convert(plan_vars, rich_data: true, @@ -206,9 +193,13 @@ def apply_ast(raw_ast, targets, options, plan_vars = {}) # This data isn't available on the target config hash config: @inventory.transport_data_get } - description = options[:description] || 'apply catalog' + @required_modules = Array(options[:required_modules]) + if @required_modules.any? + @logger.debug("Syncing only required modules: #{@required_modules.join(',')}.") + end + r = @executor.log_action(description, targets) do futures = targets.map do |target| Concurrent::Future.execute(executor: @pool) do @@ -235,9 +226,23 @@ def apply_ast(raw_ast, targets, options, plan_vars = {}) result end else + + plugin_tarball = Concurrent::Delay.new do + build_plugin_tarball do |mod| + next if @required_modules.any? && !@required_modules.include?(mod.name) + search_dirs = [] + search_dirs << mod.plugins if mod.plugins? + search_dirs << mod.pluginfacts if mod.pluginfacts? + search_dirs << mod.files if mod.files? + type_files = "#{mod.path}/types" + search_dirs << type_files if File.exist?(type_files) + search_dirs + end + end + arguments = { 'catalog' => Puppet::Pops::Types::PSensitiveType::Sensitive.new(catalog), - 'plugins' => Puppet::Pops::Types::PSensitiveType::Sensitive.new(plugins), + 'plugins' => Puppet::Pops::Types::PSensitiveType::Sensitive.new(plugins(plugin_tarball)), 'apply_settings' => @apply_settings, '_task' => catalog_apply_task.name, '_noop' => options[:noop] @@ -272,8 +277,8 @@ def apply_ast(raw_ast, targets, options, plan_vars = {}) r end - def plugins - @plugin_tarball.value || + def plugins(for_target_node) + for_target_node.value || raise(Bolt::Error.new("Failed to pack module plugins: #{@plugin_tarball.reason}", 'bolt/plugin-error')) end diff --git a/spec/bolt/applicator_spec.rb b/spec/bolt/applicator_spec.rb index 5025be1a52..42749c669b 100644 --- a/spec/bolt/applicator_spec.rb +++ b/spec/bolt/applicator_spec.rb @@ -10,6 +10,7 @@ describe Bolt::Applicator do let(:uri) { 'foobar' } + let(:plugindirs) { [] } let(:target) { inventory.get_target(uri) } let(:inventory) { Bolt::Inventory.empty } let(:executor) { Bolt::Executor.new } @@ -20,7 +21,7 @@ end let(:pdb_client) { Bolt::PuppetDB::Client.new(config) } let(:modulepath) { [Bolt::PAL::BOLTLIB_PATH, Bolt::PAL::MODULES_PATH] } - let(:applicator) { Bolt::Applicator.new(inventory, executor, modulepath, [], nil, pdb_client, nil, 2, {}) } + let(:applicator) { Bolt::Applicator.new(inventory, executor, modulepath, plugindirs, nil, pdb_client, nil, 2, {}) } let(:ast) { { 'resources' => [] } } let(:report) { @@ -115,6 +116,61 @@ let(:scope) { double('scope') } + context 'without required modules specified (default)' do + before do + allow(Logging).to receive(:logger).and_return(mock_logger) + allow(mock_logger).to receive(:[]).and_return(mock_logger) + allow(mock_logger).to receive(:'level=').with(any_args) + allow(mock_logger).to receive(:debug).with(any_args) + end + + let(:mock_logger) { instance_double("Logging.logger") } + let(:plugindirs) { modulepath } + + it 'syncs all modules' do + # + # Use a variable here instead of the Rspec let, so we can mock the logger + # + applicator = Bolt::Applicator.new(inventory, executor, modulepath, plugindirs, nil, pdb_client, nil, 2, {}) + expect(applicator).to receive(:compile).and_return(ast) + result = Bolt::Result.new(target, value: report) + allow_any_instance_of(Bolt::Transport::SSH).to receive(:batch_task).and_return(result) + allow(Bolt::ApplyResult).to receive(:puppet_missing_error).with(result).and_return(nil) + + expect(mock_logger).to receive(:debug).with(/Packing plugin/).at_least(:once) + expect(mock_logger).to_not receive(:debug).with(/Syncing only required modules/) + applicator.apply([target], :body, scope) + end + end + + context 'required modules specified' do + before do + allow(Logging).to receive(:logger).and_return(mock_logger) + allow(mock_logger).to receive(:[]).and_return(mock_logger) + allow(mock_logger).to receive(:'level=').with(any_args) + allow(mock_logger).to receive(:debug).with(any_args) + end + + let(:mock_logger) { instance_double("Logging.logger") } + let(:plugindirs) { modulepath } + + it 'syncs only required modules' do + # + # Use a variable here instead of the Rspec let, so we can mock the logger + # + applicator = Bolt::Applicator.new(inventory, executor, modulepath, plugindirs, nil, pdb_client, nil, 2, {}) + expect(applicator).to receive(:compile).and_return(ast) + result = Bolt::Result.new(target, value: report) + allow_any_instance_of(Bolt::Transport::SSH).to receive(:batch_task).and_return(result) + allow(Bolt::ApplyResult).to receive(:puppet_missing_error).with(result).and_return(nil) + + expect(mock_logger).to_not receive(:debug).with(/Packing plugin/) + expect(mock_logger).to receive(:debug).with('Syncing only required modules: just_a_module_name.') + + applicator.apply_ast(:body, [target], { required_modules: ['just_a_module_name'] }) + end + end + it 'replaces failures to find Puppet' do expect(applicator).to receive(:compile).and_return(ast) result = Bolt::Result.new(target, value: report)