diff --git a/bolt-modules/boltlib/lib/puppet/functions/resolve_references.rb b/bolt-modules/boltlib/lib/puppet/functions/resolve_references.rb index 5c8d01044a..08b6b1cc4a 100644 --- a/bolt-modules/boltlib/lib/puppet/functions/resolve_references.rb +++ b/bolt-modules/boltlib/lib/puppet/functions/resolve_references.rb @@ -26,13 +26,13 @@ def resolve_references(references) unless Puppet[:tasks] raise Puppet::ParseErrorWithIssue - .from_issue_and_stack(Bolt::PAL::Issues::PLAN_OPERATION_NOT_SUPPORTED_WHEN_COMPILING, action: 'resolve_references') + .from_issue_and_stack( + Bolt::PAL::Issues::PLAN_OPERATION_NOT_SUPPORTED_WHEN_COMPILING, + action: 'resolve_references' + ) end - references = references.merge('name' => 'all') unless references['name'] - inventory = Puppet.lookup(:bolt_inventory) - group = Bolt::Inventory::Group2.new(references, inventory.plugins) - - group.resolve_references(references) + plugins = Puppet.lookup(:bolt_inventory).plugins + plugins.resolve_references(references) end end diff --git a/bolt-modules/boltlib/spec/fixtures/modules/test/tasks/references.rb b/bolt-modules/boltlib/spec/fixtures/modules/test/tasks/references.rb new file mode 100755 index 0000000000..761c4daf2e --- /dev/null +++ b/bolt-modules/boltlib/spec/fixtures/modules/test/tasks/references.rb @@ -0,0 +1,12 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'json' + +values = { + "value" => { + "name" => "127.0.0.1" + } +} + +puts values.to_json diff --git a/bolt-modules/boltlib/spec/functions/resolve_references_spec.rb b/bolt-modules/boltlib/spec/functions/resolve_references_spec.rb index 5232aabf72..2ebab5dbb3 100644 --- a/bolt-modules/boltlib/spec/functions/resolve_references_spec.rb +++ b/bolt-modules/boltlib/spec/functions/resolve_references_spec.rb @@ -1,9 +1,61 @@ # frozen_string_literal: true require 'spec_helper' +require 'bolt/plugin' describe 'resolve_references' do - it 'resolves data' do - # TODO + include PuppetlabsSpec::Fixtures + let(:boltdir) { Bolt::Boltdir.new('./spec/fixtures') } + let(:config) { Bolt::Config.new(boltdir, {}) } + let(:pal) { Bolt::PAL.new(config.modulepath, config.hiera_config, config.boltdir.resource_types) } + let(:pdb_client) { Bolt::PuppetDB::Client.new({}) } + let(:analytics) { Bolt::Analytics::NoopClient.new } + let(:plugins) { Bolt::Plugin.setup(config, pal, pdb_client, analytics) } + let(:inventory) { Bolt::Inventory.new({}, plugins: plugins) } + let(:tasks_enabled) { true } + + let(:references) do + { + "targets" => { + "_plugin" => "task", + "task" => "test::references" + } + } + end + + let(:resolved) do + { + "targets" => { + "name" => "127.0.0.1" + } + } + end + + around(:each) do |example| + Puppet[:tasks] = tasks_enabled + Puppet.override(bolt_inventory: inventory) do + example.run + end + end + + context 'calls resolve_references' do + it 'resolves all plugin references' do + is_expected.to run.with_params(references).and_return(resolved) + end + + it 'errors when called with incorrect plugin data' do + references['targets']['_plugin'] = 'fake_plugin' + is_expected.to run.with_params(references).and_raise_error(/Unknown plugin/) + end + end + + context 'without tasks enabled' do + let(:tasks_enabled) { false } + + it 'fails and reports that resolve_references is not available' do + is_expected.to run + .with_params(references) + .and_raise_error(/Plan language function 'resolve_references' cannot be used/) + end end end diff --git a/lib/bolt/inventory/group2.rb b/lib/bolt/inventory/group2.rb index 5b1568254e..bf41716fa8 100644 --- a/lib/bolt/inventory/group2.rb +++ b/lib/bolt/inventory/group2.rb @@ -22,11 +22,11 @@ def initialize(input, plugins) @logger = Logging.logger[self] @plugins = plugins - input = resolve_top_level_references(input) if reference?(input) + input = @plugins.resolve_top_level_references(input) if @plugins.reference?(input) raise ValidationError.new("Group does not have a name", nil) unless input.key?('name') - @name = resolve_references(input['name']) + @name = @plugins.resolve_references(input['name']) raise ValidationError.new("Group name must be a String, not #{@name.inspect}", nil) unless @name.is_a?(String) raise ValidationError.new("Invalid group name #{@name}", @name) unless @name =~ NAME_REGEX @@ -37,7 +37,7 @@ def initialize(input, plugins) validate_data_keys(@input) - targets = resolve_top_level_references(input.fetch('targets', [])) + targets = @plugins.resolve_top_level_references(input.fetch('targets', [])) @unresolved_targets = {} @resolved_targets = {} @@ -67,82 +67,11 @@ def initialize(input, plugins) # them until we have a value. We don't just use resolve_references # though, since that will resolve any nested references and we want to # leave it to the group to do that lazily. - groups = resolve_top_level_references(groups) + groups = @plugins.resolve_top_level_references(groups) @groups = Array(groups).map { |g| Group2.new(g, plugins) } end - # Evaluate all _plugin references in a data structure. Leaves are - # evaluated and then their parents are evaluated with references replaced - # by their values. If the result of a reference contains more references, - # they are resolved again before continuing to ascend the tree. The final - # result will not contain any references. - def resolve_references(data) - Bolt::Util.postwalk_vals(data) do |value| - reference?(value) ? resolve_references(resolve_single_reference(value)) : value - end - end - - # Iteratively resolves "top-level" references until the result no longer - # has top-level references. A top-level reference is one which is not - # contained within another hash. It may be either the actual top-level - # result or arbitrarily nested within arrays. If parameters of the - # reference are themselves references, they will be looked. Any remaining - # references nested inside the result will *not* be evaluated once the - # top-level result is not a reference. This is used to resolve the - # `targets` and `groups` keys which are allowed to be references or - # arrays of references, but which may return data with nested references - # that should be resolved lazily. The end result will either be a single - # hash or a flat array of hashes. - def resolve_top_level_references(data) - if data.is_a?(Array) - data.flat_map { |elem| resolve_top_level_references(elem) } - elsif reference?(data) - partially_resolved = data.map do |k, v| - [k, resolve_references(v)] - end.to_h - fully_resolved = resolve_single_reference(partially_resolved) - # The top-level reference may have returned more references, so repeat the process - resolve_top_level_references(fully_resolved) - else - data - end - end - private :resolve_top_level_references - - # Evaluates a single reference. The value returned may be another - # reference. - def resolve_single_reference(reference) - plugin_name = reference['_plugin'] - begin - hook = @plugins.get_hook(plugin_name, :resolve_reference) - rescue Bolt::Plugin::PluginError => e - raise ValidationError.new(e.message, @name) - end - - begin - validate_proc = @plugins.get_hook(plugin_name, :validate_resolve_reference) - rescue Bolt::Plugin::PluginError - validate_proc = proc { |*args| } - end - - validate_proc.call(reference) - - begin - # Evaluate the plugin and then recursively evaluate any plugin returned by it. - hook.call(reference) - rescue StandardError => e - loc = "resolve_reference in #{@name}" - raise Bolt::Plugin::PluginError::ExecutionError.new(e.message, plugin_name, loc) - end - end - private :resolve_single_reference - - # Checks whether a given value is a _plugin reference - def reference?(input) - input.is_a?(Hash) && input.key?('_plugin') - end - def target_data(target_name) if @unresolved_targets.key?(target_name) target = @unresolved_targets.delete(target_name) @@ -166,9 +95,9 @@ def add_target_definition(target) raise ValidationError.new("Node entry must be a Hash, not #{target.class}", @name) end - target['name'] = resolve_references(target['name']) if target.key?('name') - target['uri'] = resolve_references(target['uri']) if target.key?('uri') - target['alias'] = resolve_references(target['alias']) if target.key?('alias') + target['name'] = @plugins.resolve_references(target['name']) if target.key?('name') + target['uri'] = @plugins.resolve_references(target['uri']) if target.key?('uri') + target['alias'] = @plugins.resolve_references(target['alias']) if target.key?('alias') t_name = target['name'] || target['uri'] @@ -294,7 +223,7 @@ def validate_group_input(input) end Bolt::Util.walk_keys(input) do |key| - if reference?(key) + if @plugins.reference?(key) raise ValidationError.new("Group keys cannot be specified as _plugin references", @name) else key @@ -357,11 +286,11 @@ def validate(used_group_names = Set.new, used_target_names = Set.new, used_alias def resolve_data_keys(data, target = nil) result = { - 'config' => resolve_references(data.fetch('config', {})), - 'vars' => resolve_references(data.fetch('vars', {})), - 'facts' => resolve_references(data.fetch('facts', {})), - 'features' => resolve_references(data.fetch('features', [])), - 'plugin_hooks' => resolve_references(data.fetch('plugin_hooks', {})) + 'config' => @plugins.resolve_references(data.fetch('config', {})), + 'vars' => @plugins.resolve_references(data.fetch('vars', {})), + 'facts' => @plugins.resolve_references(data.fetch('facts', {})), + 'features' => @plugins.resolve_references(data.fetch('features', [])), + 'plugin_hooks' => @plugins.resolve_references(data.fetch('plugin_hooks', {})) } validate_data_keys(result, target) result['features'] = Set.new(result['features'].flatten) @@ -376,13 +305,13 @@ def validate_data_keys(data, target = nil) 'features' => Array, 'plugin_hooks' => Hash }.each do |key, expected_type| - next if !data.key?(key) || data[key].is_a?(expected_type) || reference?(data[key]) + next if !data.key?(key) || data[key].is_a?(expected_type) || @plugins.reference?(data[key]) msg = +"Expected #{key} to be of type #{expected_type}, not #{data[key].class}" msg << " for target #{target}" if target raise ValidationError.new(msg, @name) end - unless reference?(data['config']) + unless @plugins.reference?(data['config']) unexpected_keys = data.fetch('config', {}).keys - CONFIG_KEYS if unexpected_keys.any? msg = +"Found unexpected key(s) #{unexpected_keys.join(', ')} in config for" diff --git a/lib/bolt/plugin.rb b/lib/bolt/plugin.rb index 034c319719..fddd1d496e 100644 --- a/lib/bolt/plugin.rb +++ b/lib/bolt/plugin.rb @@ -208,6 +208,76 @@ def by_name(plugin_name) nil end end + + # Evaluate all _plugin references in a data structure. Leaves are + # evaluated and then their parents are evaluated with references replaced + # by their values. If the result of a reference contains more references, + # they are resolved again before continuing to ascend the tree. The final + # result will not contain any references. + def resolve_references(data) + Bolt::Util.postwalk_vals(data) do |value| + reference?(value) ? resolve_references(resolve_single_reference(value)) : value + end + end + + # Iteratively resolves "top-level" references until the result no longer + # has top-level references. A top-level reference is one which is not + # contained within another hash. It may be either the actual top-level + # result or arbitrarily nested within arrays. If parameters of the + # reference are themselves references, they will be looked. Any remaining + # references nested inside the result will *not* be evaluated once the + # top-level result is not a reference. This is used to resolve the + # `targets` and `groups` keys which are allowed to be references or + # arrays of references, but which may return data with nested references + # that should be resolved lazily. The end result will either be a single + # hash or a flat array of hashes. + def resolve_top_level_references(data) + if data.is_a?(Array) + data.flat_map { |elem| resolve_top_level_references(elem) } + elsif reference?(data) + partially_resolved = data.map do |k, v| + [k, resolve_references(v)] + end.to_h + fully_resolved = resolve_single_reference(partially_resolved) + # The top-level reference may have returned more references, so repeat the process + resolve_top_level_references(fully_resolved) + else + data + end + end + + # Evaluates a single reference. The value returned may be another + # reference. + def resolve_single_reference(reference) + plugin_name = reference['_plugin'] + begin + hook = get_hook(plugin_name, :resolve_reference) + rescue Bolt::Plugin::PluginError => e + raise Bolt::ValidationError(e.message) + end + + begin + validate_proc = get_hook(plugin_name, :validate_resolve_reference) + rescue Bolt::Plugin::PluginError + validate_proc = proc { |*args| } + end + + validate_proc.call(reference) + + begin + # Evaluate the plugin and then recursively evaluate any plugin returned by it. + hook.call(reference) + rescue StandardError => e + loc = "resolve_reference in #{plugin_name}" + raise Bolt::Plugin::PluginError::ExecutionError.new(e.message, plugin_name, loc) + end + end + private :resolve_single_reference + + # Checks whether a given value is a _plugin reference + def reference?(input) + input.is_a?(Hash) && input.key?('_plugin') + end end end diff --git a/spec/bolt/inventory/inventory2_spec.rb b/spec/bolt/inventory/inventory2_spec.rb index 650e3c5610..7b2722b85e 100644 --- a/spec/bolt/inventory/inventory2_spec.rb +++ b/spec/bolt/inventory/inventory2_spec.rb @@ -169,17 +169,17 @@ def get_target(inventory, name, alia = nil) describe :validate do it 'accepts empty inventory' do - expect(Bolt::Inventory::Inventory2.new({}).validate).to be_nil + expect(Bolt::Inventory::Inventory2.new({}, plugins: plugins).validate).to be_nil end it 'accepts non-empty inventory' do - expect(Bolt::Inventory::Inventory2.new(data).validate).to be_nil + expect(Bolt::Inventory::Inventory2.new(data, plugins: plugins).validate).to be_nil end it 'fails with unnamed groups' do data = { 'groups' => [{}] } expect { - Bolt::Inventory::Inventory2.new(data).validate + Bolt::Inventory::Inventory2.new(data, plugins: plugins).validate }.to raise_error(Bolt::Inventory::ValidationError, /Group does not have a name/) end @@ -187,32 +187,32 @@ def get_target(inventory, name, alia = nil) data = { 'targets' => [{ 'name' => '' }] } expect { - Bolt::Inventory::Inventory2.new(data) + Bolt::Inventory::Inventory2.new(data, plugins: plugins) }.to raise_error(Bolt::Inventory::ValidationError, /No name or uri for target/) end it 'fails with duplicate groups' do data = { 'groups' => [{ 'name' => 'group1' }, { 'name' => 'group1' }] } expect { - Bolt::Inventory::Inventory2.new(data).validate + Bolt::Inventory::Inventory2.new(data, plugins: plugins).validate }.to raise_error(Bolt::Inventory::ValidationError, /Tried to redefine group group1/) end end describe :collect_groups do it 'finds the all group with an empty inventory' do - inventory = Bolt::Inventory::Inventory2.new({}) + inventory = Bolt::Inventory::Inventory2.new({}, plugins: plugins) expect(inventory.get_targets('all')).to eq([]) end it 'finds the all group with a non-empty inventory' do - inventory = Bolt::Inventory::Inventory2.new(data) + inventory = Bolt::Inventory::Inventory2.new(data, plugins: plugins) targets = inventory.get_targets('all') expect(targets.size).to eq(9) end it 'finds targets in a subgroup' do - inventory = Bolt::Inventory::Inventory2.new(data) + inventory = Bolt::Inventory::Inventory2.new(data, plugins: plugins) targets = inventory.get_targets('group2') target_names = targets.map(&:name) expect(target_names).to eq(%w[target6 target7 ssh://target8 target9]) @@ -220,7 +220,7 @@ def get_target(inventory, name, alia = nil) end context 'with an empty config' do - let(:inventory) { Bolt::Inventory::Inventory2.new({}, config) } + let(:inventory) { Bolt::Inventory::Inventory2.new({}, config, plugins: plugins) } let(:target) { inventory.get_targets('notarget')[0] } it 'should accept an empty file' do @@ -242,7 +242,8 @@ def get_target(inventory, name, alia = nil) 'winrm' => { 'ssl' => false, 'ssl-verify' => false - })) + }), + plugins: plugins) } let(:target) { inventory.get_targets('notarget')[0] } @@ -261,7 +262,7 @@ def get_target(inventory, name, alia = nil) describe 'get_targets' do context 'empty inventory' do - let(:inventory) { Bolt::Inventory::Inventory2.new({}, config) } + let(:inventory) { Bolt::Inventory::Inventory2.new({}, config, plugins: plugins) } it 'should parse a single target URI' do name = 'notarget' @@ -305,7 +306,7 @@ def get_target(inventory, name, alia = nil) end context 'non-empty inventory' do - let(:inventory) { Bolt::Inventory::Inventory2.new(data) } + let(:inventory) { Bolt::Inventory::Inventory2.new(data, plugins: plugins) } it 'should parse an array of target URI and group name' do targets = inventory.get_targets(%w[a group1]).map(&:name) @@ -332,7 +333,7 @@ def get_target(inventory, name, alia = nil) end context 'with data in the group' do - let(:inventory) { Bolt::Inventory::Inventory2.new(data) } + let(:inventory) { Bolt::Inventory::Inventory2.new(data, plugins: plugins) } it 'should use value from lowest target definition' do expect(get_target(inventory, 'target4').user).to eq('me') @@ -381,7 +382,7 @@ def get_target(inventory, name, alia = nil) ] } } - let(:inventory) { Bolt::Inventory::Inventory2.new(data) } + let(:inventory) { Bolt::Inventory::Inventory2.new(data, plugins: plugins) } it 'should initialize' do expect(inventory).to be @@ -429,7 +430,7 @@ def get_target(inventory, name, alia = nil) } } } - let(:inventory) { Bolt::Inventory::Inventory2.new(data) } + let(:inventory) { Bolt::Inventory::Inventory2.new(data, plugins: plugins) } it 'should return group config for string targets' do target = get_target(inventory, 'target1') @@ -451,7 +452,7 @@ def get_target(inventory, name, alia = nil) end context 'with config errors in data' do - let(:inventory) { Bolt::Inventory::Inventory2.new(data) } + let(:inventory) { Bolt::Inventory::Inventory2.new(data, plugins: plugins) } context 'host-key-check' do let(:data) { @@ -557,7 +558,7 @@ def get_target(inventory, name, alia = nil) } } } - let(:inventory) { Bolt::Inventory::Inventory2.new(data) } + let(:inventory) { Bolt::Inventory::Inventory2.new(data, plugins: plugins) } it 'should return group config for an alias' do target = get_target(inventory, 'target2', 'alias1') @@ -620,7 +621,7 @@ def common_data(transport) } } let(:conf) { Bolt::Config.default } - let(:inventory) { Bolt::Inventory::Inventory2.new(data, conf) } + let(:inventory) { Bolt::Inventory::Inventory2.new(data, conf, plugins: plugins) } it 'should not modify existing config' do get_target(inventory, 'ssh://target') @@ -695,7 +696,7 @@ def common_data(transport) describe 'get_target' do context 'empty inventory' do - let(:inventory) { Bolt::Inventory::Inventory2.new({}, config) } + let(:inventory) { Bolt::Inventory::Inventory2.new({}, config, plugins: plugins) } it 'should parse a single target URI' do name = 'notarget' @@ -718,7 +719,7 @@ def common_data(transport) end context 'non-empty inventory' do - let(:inventory) { Bolt::Inventory::Inventory2.new(data) } + let(:inventory) { Bolt::Inventory::Inventory2.new(data, plugins: plugins) } it 'a target that does not exists in inventory is created and added to the all group' do existing_target_names = inventory.get_targets('all').map(&:name) @@ -743,7 +744,7 @@ def common_data(transport) end context 'with data in the group' do - let(:inventory) { Bolt::Inventory::Inventory2.new(data) } + let(:inventory) { Bolt::Inventory::Inventory2.new(data, plugins: plugins) } it 'should use value from lowest target definition' do expect(inventory.get_target('target4').user).to eq('me') @@ -787,7 +788,7 @@ def common_data(transport) ] } } - let(:inventory) { Bolt::Inventory::Inventory2.new(data) } + let(:inventory) { Bolt::Inventory::Inventory2.new(data, plugins: plugins) } it 'should return {} for a string target' do target = inventory.get_target('target1') @@ -831,7 +832,7 @@ def common_data(transport) } } } - let(:inventory) { Bolt::Inventory::Inventory2.new(data) } + let(:inventory) { Bolt::Inventory::Inventory2.new(data, plugins: plugins) } it 'should return group config for string targets' do target = inventory.get_target('target1') @@ -880,7 +881,7 @@ def common_data(transport) } } } - let(:inventory) { Bolt::Inventory::Inventory2.new(data) } + let(:inventory) { Bolt::Inventory::Inventory2.new(data, plugins: plugins) } it 'should return group config for an alias' do target = inventory.get_target('alias1') @@ -939,7 +940,7 @@ def common_data(transport) } } let(:conf) { Bolt::Config.default } - let(:inventory) { Bolt::Inventory::Inventory2.new(data, conf) } + let(:inventory) { Bolt::Inventory::Inventory2.new(data, conf, plugins: plugins) } it 'should not modify existing config' do inventory.get_target('ssh://target') @@ -1020,7 +1021,7 @@ def common_data(transport) { 'targets' => ['localhost'] } } - let(:inventory) { Bolt::Inventory::Inventory2.new(data) } + let(:inventory) { Bolt::Inventory::Inventory2.new(data, plugins: plugins) } it 'adds magic config options' do target = get_target(inventory, 'localhost') @@ -1041,7 +1042,7 @@ def common_data(transport) } } } } - let(:inventory) { Bolt::Inventory::Inventory2.new(data) } + let(:inventory) { Bolt::Inventory::Inventory2.new(data, plugins: plugins) } it 'does not override config options' do target = get_target(inventory, 'localhost') @@ -1063,7 +1064,7 @@ def common_data(transport) } }] } } - let(:inventory) { Bolt::Inventory::Inventory2.new(data) } + let(:inventory) { Bolt::Inventory::Inventory2.new(data, plugins: plugins) } it 'does not set magic config' do target = get_target(inventory, 'localhost') expect(target.protocol).to eq('ssh') @@ -1091,7 +1092,7 @@ def common_data(transport) 'config' => { 'ssh' => { 'disconnect-timeout' => 11 } } }] } } - let(:inventory) { Bolt::Inventory::Inventory2.new(data) } + let(:inventory) { Bolt::Inventory::Inventory2.new(data, plugins: plugins) } let(:expected_options) { { "connect-timeout" => 10, "tty" => false, @@ -1201,7 +1202,7 @@ def common_data(transport) }] } } - let(:inventory) { Bolt::Inventory::Inventory2.new(data) } + let(:inventory) { Bolt::Inventory::Inventory2.new(data, plugins: plugins) } it 'adds target to a group and inherets config' do target = inventory.get_target('new-target')