command: add use-dag options for commands

For all the commands that call Initialise, we introduce a new flag:
UseDAG.

This enables DAG scheduling for evaluating datasources and locals, so
they can be referenced in any order, so long as there are no cycles or
self-referencing nodes.

`hcl2_upgrade` is a special case here, as the template is always JSON,
there cannot be any datasource, so the DAG in this case becomes
meaningless, and is not integrated in this code path.

command/build.go

@@ -102,7 +102,9 @@ func (c *BuildCommand) RunContext(buildCtx context.Context, cla *BuildArgs) int
 		return ret
 	}
 
-	diags = packerStarter.Initialize(packer.InitializeOptions{})
+	diags = packerStarter.Initialize(packer.InitializeOptions{
+		UseDAG: cla.UseDAG,
+	})
 	ret = writeDiags(c.Ui, nil, diags)
 	if ret != 0 {
 		return ret

command/cli.go

@@ -75,6 +75,12 @@ type MetaArgs struct {
 	// WarnOnUndeclared does not have a common default, as the default varies per sub-command usage.
 	// Refer to individual command FlagSets for usage.
 	WarnOnUndeclaredVar bool
+	// UseDAG specifies whether or not to use a DAG for evaluating datasources/locals
+	//
+	// It is left off by default for compatibility purposes, but this allows
+	// users to try it out if they have a need for it, or if they want to
+	// test that option for bug-reporting.
+	UseDAG bool
 }
 
 func (ba *BuildArgs) AddFlagSets(flags *flag.FlagSet) {
@@ -90,6 +96,7 @@ func (ba *BuildArgs) AddFlagSets(flags *flag.FlagSet) {
 	flags.Var(flagOnError, "on-error", "")
 
 	flags.BoolVar(&ba.MetaArgs.WarnOnUndeclaredVar, "warn-on-undeclared-var", false, "Show warnings for variable files containing undeclared variables.")
+	flags.BoolVar(&ba.MetaArgs.UseDAG, "use-dag", false, "experimental: use a DAG for evaluating datasources/locals (HCL2 only). This is not yet ready for production use.")
 
 	flags.BoolVar(&ba.ReleaseOnly, "ignore-prerelease-plugins", false, "Disable the loading of prerelease plugin binaries (x.y.z-dev).")
 
@@ -156,6 +163,7 @@ type ConsoleArgs struct {
 
 func (fa *FixArgs) AddFlagSets(flags *flag.FlagSet) {
 	flags.BoolVar(&fa.Validate, "validate", true, "")
+	flags.BoolVar(&fa.MetaArgs.UseDAG, "use-dag", false, "experimental: use a DAG for evaluating datasources/locals (HCL2 only). This is not yet ready for production use.")
 
 	fa.MetaArgs.AddFlagSets(flags)
 }
@@ -171,6 +179,7 @@ func (va *ValidateArgs) AddFlagSets(flags *flag.FlagSet) {
 	flags.BoolVar(&va.NoWarnUndeclaredVar, "no-warn-undeclared-var", false, "Ignore warnings for variable files containing undeclared variables.")
 	flags.BoolVar(&va.EvaluateDatasources, "evaluate-datasources", false, "evaluate datasources for validation (HCL2 only, may incur costs)")
 	flags.BoolVar(&va.ReleaseOnly, "ignore-prerelease-plugins", false, "Disable the loading of prerelease plugin binaries (x.y.z-dev).")
+	flags.BoolVar(&va.MetaArgs.UseDAG, "use-dag", false, "experimental: use a DAG for evaluating datasources/locals (HCL2 only). This is not yet ready for production use.")
 
 	va.MetaArgs.AddFlagSets(flags)
 }
@@ -184,6 +193,7 @@ type ValidateArgs struct {
 }
 
 func (va *InspectArgs) AddFlagSets(flags *flag.FlagSet) {
+	flags.BoolVar(&va.MetaArgs.UseDAG, "use-dag", false, "experimental: use a DAG for evaluating datasources/locals (HCL2 only). This is not yet ready for production use.")
 	va.MetaArgs.AddFlagSets(flags)
 }
 

command/console.go

@@ -63,7 +63,9 @@ func (c *ConsoleCommand) RunContext(ctx context.Context, cla *ConsoleArgs) int {
 		return ret
 	}
 
-	_ = packerStarter.Initialize(packer.InitializeOptions{})
+	_ = packerStarter.Initialize(packer.InitializeOptions{
+		UseDAG: cla.UseDAG,
+	})
 
 	// Determine if stdin is a pipe. If so, we evaluate directly.
 	if c.StdinPiped() {

command/hcl2_upgrade.go

@@ -181,7 +181,12 @@ func (c *HCL2UpgradeCommand) RunContext(_ context.Context, cla *HCL2UpgradeArgs)
 	}
 
 	core := hdl.(*packer.Core)
-	if err := core.Initialize(packer.InitializeOptions{}); err != nil {
+	if err := core.Initialize(packer.InitializeOptions{
+		// Note: this is always false here as the DAG is only usable for
+		// HCL2 configs, so since the command only works on JSON templates,
+		// we can safely leave it off .
+		UseDAG: false,
+	}); err != nil {
 		c.Ui.Error(fmt.Sprintf("Ignoring following initialization error: %v", err))
 	}
 	tpl := core.Template

command/inspect.go

@@ -49,7 +49,9 @@ func (c *InspectCommand) RunContext(ctx context.Context, cla *InspectArgs) int {
 	}
 
 	// here we ignore init diags to allow unknown variables to be used
-	_ = packerStarter.Initialize(packer.InitializeOptions{})
+	_ = packerStarter.Initialize(packer.InitializeOptions{
+		UseDAG: cla.UseDAG,
+	})
 
 	return packerStarter.InspectConfig(packer.InspectConfigOptions{
 		Ui: c.Ui,

command/validate.go

@@ -78,6 +78,7 @@ func (c *ValidateCommand) RunContext(ctx context.Context, cla *ValidateArgs) int
 
 	diags = packerStarter.Initialize(packer.InitializeOptions{
 		SkipDatasourcesExecution: !cla.EvaluateDatasources,
+		UseDAG:                   cla.UseDAG,
 	})
 	ret = writeDiags(c.Ui, nil, diags)
 	if ret != 0 {