Test suite cleanup

packer_test/lib/base.go → packer_test/common/base.go

@@ -1,4 +1,4 @@
-package lib
+package common
 
 import (
 	"fmt"

packer_test/lib/gadgets.go → packer_test/common/check/gadgets.go

@@ -1,4 +1,4 @@
-package lib
+package check
 
 import (
 	"fmt"
@@ -125,6 +125,10 @@ func Grep(expression string, opts ...GrepOpts) Checker {
 	return pc
 }
 
+func GrepInverted(expression string, opts ...GrepOpts) Checker {
+	return Grep(expression, append(opts, GrepInvert)...)
+}
+
 type PluginVersionTuple struct {
 	Source  string
 	Version *version.Version

packer_test/lib/pipe_checker.go → packer_test/common/check/pipe_checker.go

@@ -1,4 +1,4 @@
-package lib
+package check
 
 import (
 	"fmt"

packer_test/lib/commands.go → packer_test/common/commands.go

@@ -1,24 +1,27 @@
-package lib
+package common
 
 import (
 	"fmt"
 	"os"
 	"os/exec"
 	"strings"
 	"testing"
+
+	"github.com/hashicorp/packer/packer_test/common/check"
 )
 
 type packerCommand struct {
-	runs       int
-	packerPath string
-	args       []string
-	env        map[string]string
-	stdin      string
-	stderr     *strings.Builder
-	stdout     *strings.Builder
-	workdir    string
-	err        error
-	t          *testing.T
+	runs         int
+	packerPath   string
+	args         []string
+	env          map[string]string
+	stdin        string
+	stderr       *strings.Builder
+	stdout       *strings.Builder
+	workdir      string
+	err          error
+	t            *testing.T
+	fatalfAssert bool
 }
 
 // PackerCommand creates a skeleton of packer command with the ability to execute gadgets on the outputs of the command.
@@ -106,13 +109,23 @@ func (pc *packerCommand) Stdin(in string) *packerCommand {
 	return pc
 }
 
+// SetAssertFatal allows changing how Assert behaves when reporting an error.
+//
+// By default Assert will invoke t.Errorf with the error details, but this can be
+// changed to a t.Fatalf so that if the assertion fails, the test invoking it will
+// also immediately fail and stop execution.
+func (pc *packerCommand) SetAssertFatal() *packerCommand {
+	pc.fatalfAssert = true
+	return pc
+}
+
 // Run executes the packer command with the args/env requested and returns the
 // output streams (stdout, stderr)
 //
 // Note: while originally "Run" was designed to be idempotent, with the
 // introduction of multiple runs for a command, this is not the case anymore
 // and the function should not be considered thread-safe anymore.
-func (pc *packerCommand) Run() (string, string, error) {
+func (pc *packerCommand) run() (string, string, error) {
 	if pc.runs <= 0 {
 		return pc.stdout.String(), pc.stderr.String(), pc.err
 	}
@@ -139,24 +152,40 @@ func (pc *packerCommand) Run() (string, string, error) {
 	pc.err = cmd.Run()
 
 	// Check that the command didn't panic, and if it did, we can immediately error
-	panicErr := PanicCheck{}.Check(pc.stdout.String(), pc.stderr.String(), pc.err)
+	panicErr := check.PanicCheck{}.Check(pc.stdout.String(), pc.stderr.String(), pc.err)
 	if panicErr != nil {
 		pc.t.Fatalf("Packer panicked during execution: %s", panicErr)
 	}
 
 	return pc.stdout.String(), pc.stderr.String(), pc.err
 }
 
-func (pc *packerCommand) Assert(checks ...Checker) {
+// Output returns the results of the latest Run that was executed.
+//
+// In general there is only one run of the command, but as it can be changed
+// through the Runs function, only the latest run will be returned.
+//
+// If the command was not run (through Assert), this will make the test fail
+// immediately.
+func (pc *packerCommand) Output() (string, string, error) {
+	if pc.runs > 0 {
+		pc.t.Fatalf("command was not run, invoke Assert first, then Output.")
+	}
+
+	return pc.stdout.String(), pc.stderr.String(), pc.err
+}
+
+func (pc *packerCommand) Assert(checks ...check.Checker) {
 	attempt := 0
 	for pc.runs > 0 {
 		attempt++
-		stdout, stderr, err := pc.Run()
+		stdout, stderr, err := pc.run()
 
-		for _, check := range checks {
-			checkErr := check.Check(stdout, stderr, err)
+		for _, checker := range checks {
+			checkErr := checker.Check(stdout, stderr, err)
 			if checkErr != nil {
-				checkerName := InferName(check)
+				checkerName := check.InferName(checker)
+
 				pc.t.Errorf("check %q failed: %s", checkerName, checkErr)
 			}
 		}
@@ -167,6 +196,10 @@ func (pc *packerCommand) Assert(checks ...Checker) {
 			pc.t.Logf("dumping stdout: %s", stdout)
 			pc.t.Logf("dumping stdout: %s", stderr)
 
+			if pc.fatalfAssert {
+				pc.t.Fatalf("stopping test now because of failures reported")
+			}
+
 			break
 		}
 	}

packer_test/lib/fs.go → packer_test/common/fs.go

@@ -1,4 +1,4 @@
-package lib
+package common
 
 import (
 	"crypto/sha256"

packer_test/lib/plugin.go → packer_test/common/plugin.go

@@ -1,4 +1,4 @@
-package lib
+package common
 
 import (
 	"fmt"
@@ -7,39 +7,13 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
-	"sync"
 	"testing"
 
 	"github.com/hashicorp/go-version"
 	"github.com/hashicorp/packer-plugin-sdk/plugin"
+	"github.com/hashicorp/packer/packer_test/common/check"
 )
 
-type compiledPlugins struct {
-	pluginVersions map[string]string
-	mutex          sync.Mutex
-}
-
-func (ts *PackerTestSuite) StorePluginVersion(pluginVersion, path string) {
-	ts.compiledPlugins.mutex.Lock()
-	defer ts.compiledPlugins.mutex.Unlock()
-	if ts.compiledPlugins.pluginVersions == nil {
-		ts.compiledPlugins.pluginVersions = map[string]string{}
-	}
-
-	ts.compiledPlugins.pluginVersions[pluginVersion] = path
-}
-
-func (ts *PackerTestSuite) LoadPluginVersion(pluginVersion string) (string, bool) {
-	ts.compiledPlugins.mutex.Lock()
-	defer ts.compiledPlugins.mutex.Unlock()
-	if ts.compiledPlugins.pluginVersions == nil {
-		ts.compiledPlugins.pluginVersions = map[string]string{}
-	}
-
-	path, ok := ts.compiledPlugins.pluginVersions[pluginVersion]
-	return path, ok
-}
-
 // LDFlags compiles the ldflags for the plugin to compile based on the information provided.
 func LDFlags(version *version.Version) string {
 	pluginPackage := "github.com/hashicorp/packer-plugin-tester"
@@ -88,9 +62,24 @@ func ExpectedInstalledName(versionStr string) string {
 		runtime.GOOS, runtime.GOARCH, ext)
 }
 
-// BuildSimplePlugin creates a plugin that essentially does nothing.
+// GetPluginPath gets the path for a pre-compiled plugin in the current test suite.
 //
-// The plugin's code is contained in a subdirectory of this, and lets us
+// The version only is needed, as the path to a compiled version of the tester
+// plugin will be returned, so it can be installed after the fact.
+//
+// If the version requested does not exist, the function will panic.
+func (ts *PackerTestSuite) GetPluginPath(t *testing.T, version string) string {
+	path, ok := ts.compiledPlugins.Load(version)
+	if !ok {
+		t.Fatalf("tester plugin in version %q was not built, either build it during suite init, or with BuildTestPlugin", version)
+	}
+
+	return path.(string)
+}
+
+// CompilePlugin builds a tester plugin with the specified version.
+//
+// The plugin's code is contained in a subdirectory of this file, and lets us
 // change the attributes of the plugin binary itself, like the SDK version,
 // the plugin's version, etc.
 //
@@ -100,16 +89,24 @@ func ExpectedInstalledName(versionStr string) string {
 //
 // The path to the plugin is returned, it won't be removed automatically
 // though, deletion is the caller's responsibility.
-func (ts *PackerTestSuite) BuildSimplePlugin(versionString string, t *testing.T) string {
-	// Only build plugin binary if not already done beforehand
-	path, ok := ts.LoadPluginVersion(versionString)
-	if ok {
-		return path
+//
+// Note: each tester plugin may only be compiled once for a specific version in
+// a test suite. The version may include core (mandatory), pre-release and
+// metadata. Unlike Packer core, metadata does matter for the version being built.
+func (ts *PackerTestSuite) CompilePlugin(t *testing.T, versionString string) {
+	// Fail to build plugin if already built.
+	//
+	// Especially with customisations being a thing, relying on cache to get and
+	// build a plugin at once means that the function is not idempotent anymore,
+	// and therefore we cannot rely on it being called twice and producing the
+	// same result, so we forbid it.
+	if _, ok := ts.compiledPlugins.Load(versionString); ok {
+		t.Fatalf("plugin version %q was already built, use GetTestPlugin instead", versionString)
 	}
 
 	v := version.Must(version.NewSemver(versionString))
 
-	t.Logf("Building plugin in version %v", v)
+	t.Logf("Building tester plugin in version %v", v)
 
 	testDir, err := currentDir()
 	if err != nil {
@@ -125,9 +122,7 @@ func (ts *PackerTestSuite) BuildSimplePlugin(versionString string, t *testing.T)
 		t.Fatalf("failed to compile plugin binary: %s\ncompiler logs: %s", err, logs)
 	}
 
-	ts.StorePluginVersion(v.String(), outBin)
-
-	return outBin
+	ts.compiledPlugins.Store(v.String(), outBin)
 }
 
 // MakePluginDir installs a list of plugins into a temporary directory and returns its path
@@ -136,11 +131,14 @@ func (ts *PackerTestSuite) BuildSimplePlugin(versionString string, t *testing.T)
 // packer will be able to use that directory for running its functions.
 //
 // Deletion of the directory is the caller's responsibility.
+//
+// Note: all of the plugin versions specified to be installed in this plugin directory
+// must have been compiled beforehand.
 func (ts *PackerTestSuite) MakePluginDir(pluginVersions ...string) (pluginTempDir string, cleanup func()) {
 	t := ts.T()
 
-	for _, ver := range pluginVersions {
-		ts.BuildSimplePlugin(ver, t)
+	for _, version := range pluginVersions {
+		_ = ts.GetPluginPath(t, version)
 	}
 
 	var err error
@@ -160,13 +158,10 @@ func (ts *PackerTestSuite) MakePluginDir(pluginVersions ...string) (pluginTempDi
 	}
 
 	for _, pluginVersion := range pluginVersions {
-		path, ok := ts.LoadPluginVersion(pluginVersion)
-		if !ok {
-			err = fmt.Errorf("failed to get path to version %q, was it compiled?", pluginVersion)
-		}
+		path := ts.GetPluginPath(t, pluginVersion)
 		cmd := ts.PackerCommand().SetArgs("plugins", "install", "--path", path, "github.com/hashicorp/tester").AddEnv("PACKER_PLUGIN_PATH", pluginTempDir)
-		cmd.Assert(MustSucceed())
-		out, stderr, cmdErr := cmd.Run()
+		cmd.Assert(check.MustSucceed())
+		out, stderr, cmdErr := cmd.run()
 		if cmdErr != nil {
 			err = fmt.Errorf("failed to install tester plugin version %q: %s\nCommand stdout: %s\nCommand stderr: %s", pluginVersion, err, out, stderr)
 			return

packer_test/lib/suite.go → packer_test/common/suite.go

@@ -1,4 +1,4 @@
-package lib
+package common
 
 import (
 	"fmt"
@@ -27,14 +27,14 @@ type PackerTestSuite struct {
 	// we may have multiple suites that exist, each with its own repo of
 	// plugins compiled for the purposes of the test, so as they all run
 	// within the same process space, they should be separate instances.
-	compiledPlugins compiledPlugins
+	compiledPlugins sync.Map
 }
 
 func (ts *PackerTestSuite) buildPluginVersion(waitgroup *sync.WaitGroup, versionString string, t *testing.T) {
 	waitgroup.Add(1)
 	go func() {
 		defer waitgroup.Done()
-		ts.BuildSimplePlugin(versionString, t)
+		ts.CompilePlugin(t, versionString)
 	}()
 }
 
@@ -62,18 +62,17 @@ func (ts *PackerTestSuite) SkipNoAcc() {
 	}
 }
 
-func PackerCoreSuite(t *testing.T) (*PackerTestSuite, func()) {
-	ts := &PackerTestSuite{}
+func InitBaseSuite(t *testing.T) (*PackerTestSuite, func()) {
+	ts := &PackerTestSuite{
+		compiledPlugins: sync.Map{},
+	}
 
 	tempDir, err := os.MkdirTemp("", "packer-core-acc-test-")
 	if err != nil {
 		panic(fmt.Sprintf("failed to create temporary directory for compiled plugins: %s", err))
 	}
 	ts.pluginsDirectory = tempDir
 
-	defer func() {
-	}()
-
 	packerPath := os.Getenv("PACKER_CUSTOM_PATH")
 	if packerPath == "" {
 		var err error

packer_test/core_tests/local_eval_test.go

@@ -3,7 +3,7 @@ package core_test
 import (
 	"fmt"
 
-	"github.com/hashicorp/packer/packer_test/lib"
+	"github.com/hashicorp/packer/packer_test/common/check"
 )
 
 func (ts *PackerCoreTestSuite) TestEvalLocalsOrder() {
@@ -16,8 +16,8 @@ func (ts *PackerCoreTestSuite) TestEvalLocalsOrder() {
 		Runs(10).
 		Stdin("local.test_local\n").
 		SetArgs("console", "./templates/locals_no_order.pkr.hcl").
-		Assert(lib.MustSucceed(),
-			lib.Grep("\\[\\]", lib.GrepStdout, lib.GrepInvert))
+		Assert(check.MustSucceed(),
+			check.GrepInverted("\\[\\]", check.GrepStdout))
 }
 
 func (ts *PackerCoreTestSuite) TestLocalDuplicates() {
@@ -28,9 +28,9 @@ func (ts *PackerCoreTestSuite) TestLocalDuplicates() {
 		ts.Run(fmt.Sprintf("duplicate local detection with %s command - expect error", cmd), func() {
 			ts.PackerCommand().UsePluginDir(pluginDir).
 				SetArgs(cmd, "./templates/locals_duplicate.pkr.hcl").
-				Assert(lib.MustFail(),
-					lib.Grep("Duplicate local definition"),
-					lib.Grep("Local variable \"test\" is defined twice"))
+				Assert(check.MustFail(),
+					check.Grep("Duplicate local definition"),
+					check.Grep("Local variable \"test\" is defined twice"))
 		})
 	}
 }