(puppetlabsGH-1244) Serialize plan variables as pcore

There are 4 custom types in the Bolt plan language that we want to make
available inside apply blocks: Target, Result, ResultSet, ApplyResult.
For each of the *Result* objects this is accomplished by serializing the
plan variables to the apply block as pcore instead of as json. We also
needed to add pcore_init methods to the implementation classes of the
objects, because the pcore deserializer will by default call
`initialize` on the class and pass in the values of the attributes of
the type, which doesn't match the signature of the actual
implementations. Using `_pcore_init_hash` and `_pcore_init_from_hash`
lets us define what data gets serialized and how it gets deserialized
for those types.

Target objects are trickier for a few reasons:
1. They must have an associate inventory
2. ...

Stil TODO:
- [] Future proof
- [] Tests
- [] Docs
- [] Threading

.rubocop.yml

@@ -95,3 +95,7 @@ Lint/HandleExceptions:
 # Enforce LF line endings, even when on Windows
 Layout/EndOfLine:
   EnforcedStyle: lf
+
+Lint/AmbiguousOperator:
+  Exclude:
+    - lib/bolt/apply_target.rb

bolt-modules/boltlib/lib/puppet/datatypes/applytarget.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+Puppet::DataTypes.create_type('ApplyTarget') do
+  load_file('bolt/apply_target')
+  interface <<-PUPPET
+    attributes => {
+      'target_hash' => Hash
+    },
+    functions => {
+      uri => Callable[[], String[1]],
+      name => Callable[[], String[1]],
+      target_alias => Callable[[], Optional[String]],
+      config => Callable[[], Optional[Hash[String[1], Data]]],
+      vars => Callable[[], Optional[Hash[String[1], Data]]],
+      facts => Callable[[], Optional[Hash[String[1], Data]]],
+      features => Callable[[], Optional[Array[String[1]]]],
+      plugin_hooks => Callable[[], Optional[Hash[String[1], Data]]],
+      safe_name => Callable[[], String[1]],
+      host => Callable[[], Optional[String]],
+      password => Callable[[], Optional[String[1]]],
+      port => Callable[[], Optional[Integer]],
+      protocol => Callable[[], Optional[String[1]]],
+      user => Callable[[], Optional[String[1]]],
+    }
+  PUPPET
+
+  implementation_class Bolt::ApplyTarget
+end

lib/bolt/applicator.rb

@@ -8,6 +8,8 @@
 require 'bolt/error'
 require 'bolt/task'
 require 'bolt/apply_result'
+require 'bolt/apply_target'
+require 'bolt/util'
 require 'bolt/util/puppet_log_level'
 
 module Bolt
@@ -83,7 +85,37 @@ def query_resources_task
     def compile(target, ast, plan_vars)
       trusted = Puppet::Context::TrustedInformation.new('local', target.name, {})
       facts = @inventory.facts(target).merge('bolt' => true)
+      # Convert all targets to ApplyTargets
+      # This needs to happen here so we can serialize *Result* objects as pcore
+      # types, which contain targets
+      vars = @inventory.vars(target).merge(plan_vars)
+
+      # TODO: How much of a performance hit is this?
+      vars = Bolt::Util.walk_vals(vars) do |var|
+        if var.is_a?(Bolt::Target2)
+          Bolt::ApplyTarget.new(var.detail.merge(var.to_h))
+        elsif var.is_a?(Bolt::Result)
+          Bolt::Result.from_apply_block(var)
+        elsif var.is_a?(Bolt::ResultSet)
+          Bolt::ResultSet.from_apply_block(var)
+        elsif var.is_a?(Bolt::ApplyResult)
+          Bolt::ApplyResult.from_apply_block(var)
+        else
+          var
+        end
+      end
 
+      # TODO: How do we make loaders available here with concurrent threads?
+      Puppet.lookup(:loaders).private_environment_loader.load(:type, 'applytarget')
+      Puppet.lookup(:loaders).private_environment_loader.load(:type, 'result')
+      Puppet.lookup(:loaders).private_environment_loader.load(:type, 'resultset')
+      Puppet.lookup(:loaders).private_environment_loader.load(:type, 'applyresult')
+      # Serialize as pcore for *Result* objects
+      vars = Puppet::Pops::Serialization::ToDataConverter.convert(vars,
+                                                                  rich_data: true,
+                                                                  symbol_as_string: true,
+                                                                  type_by_reference: true,
+                                                                  local_reference: false)
       catalog_input = {
         code_ast: ast,
         modulepath: @modulepath,
@@ -92,7 +124,7 @@ def compile(target, ast, plan_vars)
         target: {
           name: target.name,
           facts: facts,
-          variables: @inventory.vars(target).merge(plan_vars),
+          variables: vars,
           trusted: trusted.to_h
         },
         inventory: @inventory.data_hash
@@ -181,7 +213,7 @@ def apply_ast(raw_ast, targets, options, plan_vars = {})
 
       r = @executor.log_action('apply catalog', targets) do
         futures = targets.map do |target|
-          Concurrent::Future.execute(executor: @pool) do
+          Concurrent::Future.execute(executor: :immediate) do
             @executor.with_node_logging("Compiling manifest block", [target]) do
               compile(target, ast, plan_vars)
             end

lib/bolt/apply_result.rb

@@ -85,6 +85,27 @@ def self.from_task_result(result)
       end
     end
 
+    def self.from_apply_block(applyresult)
+      target = Bolt::ApplyTarget.new(applyresult.target.detail.merge(applyresult.target.to_h))
+      new(target, error: applyresult.error, report: applyresult.report)
+    end
+
+    def self._pcore_init_from_hash(init_hash); end
+
+    def _pcore_init_from_hash(init_hash)
+      opts = init_hash.reject { |k, _v| k == 'target' }
+      initialize(init_hash['target'], opts.transform_keys(&:to_sym))
+    end
+
+    def _pcore_init_hash
+      { 'target' => @target,
+        'error' => @value['_error'],
+        'message' => @value['_output'],
+        'value' => @value,
+        'action' => @action,
+        'report' => @value['report'] }
+    end
+
     def initialize(target, error: nil, report: nil)
       @target = target
       @value = {}

lib/bolt/apply_target.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Bolt
+  # Alt: ReadOnlyTarget
+  class ApplyTarget
+    # Should this be memoized instead?
+    ATTRIBUTES ||= %i[uri name target_alias config vars facts features
+                      plugin_hooks safe_name host password port protocol user].freeze
+
+    attr_reader *ATTRIBUTES, :target_hash
+
+    # Puppet calls this method when it needs an instance of this type
+    def self.from_asserted_hash(target_hash)
+      new(target_hash)
+    end
+
+    def self.from_asserted_args(target_hash)
+      new(target_hash)
+    end
+
+    def initialize(target_hash)
+      # TODO: Do we actually need this?
+      @target_hash = target_hash
+      ATTRIBUTES.each do |attr|
+        instance_variable_set("@#{attr}", target_hash[attr.to_s])
+      end
+    end
+  end
+end

lib/bolt/catalog.rb

@@ -1,7 +1,8 @@
 # frozen_string_literal: true
 
+require 'bolt/apply_target'
 require 'bolt/config'
-require 'bolt/inventory'
+require 'bolt/inventory/inventory2'
 require 'bolt/pal'
 require 'bolt/puppetdb'
 require 'bolt/util'
@@ -50,32 +51,28 @@ def generate_ast(code, filename = nil)
       end
     end
 
-    def setup_inventory(inventory)
-      config = Bolt::Config.default
-      config.overwrite_transport_data(inventory['config']['transport'],
-                                      Bolt::Util.symbolize_top_level_keys(inventory['config']['transports']))
-
-      Bolt::Inventory.new(inventory['data'],
-                          config,
-                          Bolt::Util.symbolize_top_level_keys(inventory['target_hash']))
-    end
-
     def compile_catalog(request)
       pal_main = request['code_ast'] || request['code_string']
       target = request['target']
       pdb_client = Bolt::PuppetDB::Client.new(Bolt::PuppetDB::Config.new(request['pdb_config']))
       options = request['puppet_config'] || {}
+
       with_puppet_settings(request['hiera_config']) do
         Puppet[:rich_data] = true
         Puppet[:node_name_value] = target['name']
         Puppet::Pal.in_tmp_environment('bolt_catalog',
                                        modulepath: request["modulepath"] || [],
-                                       facts: target["facts"] || {},
-                                       variables: target["variables"] || {}) do |pal|
-          Puppet.override(bolt_pdb_client: pdb_client,
-                          bolt_inventory: setup_inventory(request['inventory'])) do
+                                       facts: target["facts"] || {}) do |pal|
+          # TODO: What do we use this for?
+          Puppet.override(bolt_pdb_client: pdb_client) do
             Puppet.lookup(:pal_current_node).trusted_data = target['trusted']
             pal.with_catalog_compiler do |compiler|
+              # Add the variables to the compiler
+              # This has to happen inside the catalog compiler so that loaders
+              # are available to load custom types
+              vars = Puppet::Pops::Serialization::FromDataConverter.convert(target['variables'])
+              pal.send(:add_variables, compiler.send(:topscope), vars)
+
               # Configure language strictness in the CatalogCompiler. We want Bolt to be able
               # to compile most Puppet 4+ manifests, so we default to allowing deprecated functions.
               Puppet[:strict] = options['strict'] || :warning

lib/bolt/result.rb

@@ -73,6 +73,32 @@ def self.from_asserted_args(target, value)
       new(target, value: value)
     end
 
+    def self.from_apply_block(result)
+      target = Bolt::ApplyTarget.new(result.target.detail.merge(result.target.to_h))
+      new(target,
+          error: result.error,
+          message: result.message,
+          value: result.value,
+          action: result.action,
+          object: result.object)
+    end
+
+    def self._pcore_init_from_hash(init_hash); end
+
+    def _pcore_init_from_hash(init_hash)
+      opts = init_hash.reject { |k, _v| k == 'target' }
+      initialize(init_hash['target'], opts.transform_keys(&:to_sym))
+    end
+
+    def _pcore_init_hash
+      { 'target' => @target,
+        'error' => @value['_error'],
+        'message' => @value['_output'],
+        'value' => @value,
+        'action' => @action,
+        'object' => @object }
+    end
+
     def initialize(target, error: nil, message: nil, value: nil, action: nil, object: nil)
       @target = target
       @value = value || {}

lib/bolt/result_set.rb

@@ -12,6 +12,23 @@ def self.include_iterable
       include(Puppet::Pops::Types::IteratorProducer)
     end
 
+    def self.from_apply_block(resultset)
+      results = resultset.results.map do |result|
+        Bolt::Result.from_apply_block(result)
+      end
+      new(results)
+    end
+
+    def self._pcore_init_from_hash(init_hash); end
+
+    def _pcore_init_from_hash(init_hash)
+      initialize(init_hash['results'])
+    end
+
+    def _pcore_init_hash
+      { 'results' => @results }
+    end
+
     def iterator
       if Object.const_defined?(:Puppet) && Puppet.const_defined?(:Pops) &&
          self.class.included_modules.include?(Puppet::Pops::Types::Iterable)