feat!: Install tools concurrently, display progress in spinner (#7)

Spinner now works in verbose mode, no spinner animation if stderr is not a tty

BREAKING CHANGE: Cache.Install and all methods of cache.Go now require a
context as the first argument. Shed.Install now returns a
client.InstallSet which will perform the installation.

cache/cache.go

@@ -5,6 +5,7 @@
 package cache
 
 import (
+	"context"
 	"io/ioutil"
 	"os"
 	"path/filepath"
@@ -88,15 +89,24 @@ func (c *Cache) toolsDir() string {
 // an error will be returned. If t.Version is empty, then the latest version
 // of the tool will be installed. The returned tool will have Version set
 // to the version that was installed.
-func (c *Cache) Install(t tool.Tool) (tool.Tool, error) {
+//
+// The provided context is used to terminate the install if the context becomes
+// done before the install completes on its own.
+func (c *Cache) Install(ctx context.Context, t tool.Tool) (tool.Tool, error) {
+	select {
+	case <-ctx.Done():
+		return t, ctx.Err()
+	default:
+	}
+
 	// Make sure import path is set as it's required for download
 	if t.ImportPath == "" {
 		return t, errors.New("import path is required on module")
 	}
 
 	// Download step
 
-	downloadedTool, err := c.download(t)
+	downloadedTool, err := c.download(ctx, t)
 	if err != nil {
 		return t, errors.WithMessagef(err, "failed to download tool: %s", t)
 	}
@@ -125,7 +135,7 @@ func (c *Cache) Install(t tool.Tool) (tool.Tool, error) {
 		return downloadedTool, nil
 	}
 
-	err = c.goClient.Build(downloadedTool.ImportPath, binPath, binDir)
+	err = c.goClient.Build(ctx, downloadedTool.ImportPath, binPath, binDir)
 	if err != nil {
 		return downloadedTool, errors.WithMessagef(err, "failed to build tool: %s", downloadedTool)
 	}
@@ -146,7 +156,7 @@ func (c *Cache) Install(t tool.Tool) (tool.Tool, error) {
 // For example if the import path is golang.org/x/tools/cmd/stringer then download will create
 // BASE_DIR/golang.org/x/tools/cmd/stringer@VERSION/go.mod where BASE_DIR is the baseDir parameter
 // and VERSION is the version of the tool (either explicit or resolved).
-func (c *Cache) download(t tool.Tool) (tool.Tool, error) {
+func (c *Cache) download(ctx context.Context, t tool.Tool) (tool.Tool, error) {
 	// Get the path to where the tool will be installed
 	// This is where the go.mod file will be
 	fp, err := t.Filepath()
@@ -227,7 +237,7 @@ func (c *Cache) download(t tool.Tool) (tool.Tool, error) {
 		// go get so we don't need to reinvent the module resolution & downloading.
 		// Also we can reuse an existing download that's already cached.
 
-		err = c.goClient.GetD(t.Module(), modDir)
+		err = c.goClient.GetD(ctx, t.Module(), modDir)
 		if err != nil {
 			return t, err
 		}
@@ -264,7 +274,7 @@ func (c *Cache) download(t tool.Tool) (tool.Tool, error) {
 
 	// Download the module source. This will do the heavy lifting to figure out
 	// the correct version.
-	err = c.goClient.GetD(t.Module(), modDir)
+	err = c.goClient.GetD(ctx, t.Module(), modDir)
 	if err != nil {
 		return t, err
 	}

cache/go.go

@@ -2,6 +2,7 @@ package cache
 
 import (
 	"bytes"
+	"context"
 	"go/build"
 	"io/ioutil"
 	"os"
@@ -48,19 +49,25 @@ func createGoModFile(mod, dir string) error {
 	return nil
 }
 
-// Go represents the core functionality provided by the go command. It allows for downloading
-// and building of modules.
+// Go represents the core functionality provided by the go command.
+// It allows for downloading and building of modules.
 type Go interface {
 	// Build builds pkg and outputs the binary at outPath. dir is used as the working directory
 	// when building. pkg must be a valid import path.
 	// Build functions like 'go build -o'.
-	Build(pkg, outPath, dir string) error
+	//
+	// The provided context is used to terminate the build if the context becomes
+	// done before the build completes on its own.
+	Build(ctx context.Context, pkg, outPath, dir string) error
 	// GetD downloads the source code for the module mod. dir is used as the working directory
 	// and is expected to contain a go.mod file which will be updated with the installed module.
 	// mod must be a valid module name, that is an import path, optionally with a version.
 	// If no version is provided, the latest version will be downloaded.
 	// GetD functions like 'got get -d' in module aware mode.
-	GetD(mod, dir string) error
+	//
+	// The provided context is used to terminate the download if the context becomes
+	// done before the download completes on its own.
+	GetD(ctx context.Context, mod, dir string) error
 }
 
 // realGo is the main implementation of the Go interface.
@@ -72,16 +79,16 @@ func NewGo() Go {
 	return realGo{}
 }
 
-func (realGo) Build(pkg, outPath, dir string) error {
-	return execGo(dir, "build", "-o", outPath, pkg)
+func (realGo) Build(ctx context.Context, pkg, outPath, dir string) error {
+	return execGo(ctx, dir, "build", "-o", outPath, pkg)
 }
 
-func (realGo) GetD(mod, dir string) error {
-	return execGo(dir, "get", "-d", mod)
+func (realGo) GetD(ctx context.Context, mod, dir string) error {
+	return execGo(ctx, dir, "get", "-d", mod)
 }
 
-func execGo(dir string, args ...string) error {
-	cmd := exec.Command("go", args...)
+func execGo(ctx context.Context, dir string, args ...string) error {
+	cmd := exec.CommandContext(ctx, "go", args...)
 	cmd.Dir = dir
 	stderr := &bytes.Buffer{}
 	cmd.Stderr = stderr
@@ -142,7 +149,7 @@ func NewMockGo(tools map[string]map[string]string) (Go, error) {
 	return &mockGo{registry: registry}, nil
 }
 
-func (mg *mockGo) Build(pkg, outPath, dir string) error {
+func (mg *mockGo) Build(ctx context.Context, pkg, outPath, dir string) error {
 	if _, ok := mg.registry[pkg]; !ok {
 		return errors.Errorf("unknown package %s", pkg)
 	}
@@ -157,7 +164,7 @@ func (mg *mockGo) Build(pkg, outPath, dir string) error {
 	return nil
 }
 
-func (mg *mockGo) GetD(mod, dir string) error {
+func (mg *mockGo) GetD(ctx context.Context, mod, dir string) error {
 	t, err := tool.ParseLax(mod)
 	if err != nil {
 		return err

client/client.go

@@ -2,10 +2,10 @@
 package client
 
 import (
+	"context"
 	"io/ioutil"
 	"os"
 	"path/filepath"
-	"sort"
 
 	"github.com/getshiphub/shed/cache"
 	"github.com/getshiphub/shed/lockfile"
@@ -115,14 +115,18 @@ func (s *Shed) writeLockfile() error {
 	return nil
 }
 
-// Install installs zero or more given tools and add them to the lockfile.
-// It also checks if any tools in the lockfile are not installed and installs
-// them if so.
+// Install computes a set of tools that should be installed. It can be given zero or
+// more tools as arguments. These will be unioned with the tools in the lockfile
+// to produce a final set of tools to install. Install will return an InstallSet instance
+// which can be used to perform the actual installation.
+//
+// Install does not modify any state, therefore, if you wish to abort the install simply
+// discard the returned InstallSet.
 //
 // If a tool name is provided with a version and the same tool already exists in the
 // lockfile with a different version, then Install will return an error, unless allowUpdates
 // is set in which case the given tool version will overwrite the one in the lockfile.
-func (s *Shed) Install(allowUpdates bool, toolNames ...string) error {
+func (s *Shed) Install(allowUpdates bool, toolNames ...string) (*InstallSet, error) {
 	// Collect all the tools that need to be installed.
 	// Merge the given tools with what exists in the lockfile.
 	seenTools := make(map[string]bool)
@@ -155,7 +159,7 @@ func (s *Shed) Install(allowUpdates bool, toolNames ...string) error {
 		tools = append(tools, t)
 	}
 	if len(errs) > 0 {
-		return errs
+		return nil, errs
 	}
 
 	// Take union with lockfile
@@ -166,22 +170,75 @@ func (s *Shed) Install(allowUpdates bool, toolNames ...string) error {
 			tools = append(tools, t)
 		}
 	}
+	return &InstallSet{s: s, tools: tools}, nil
+}
 
-	// Sort the tools so they are always installed in the same order
-	sort.Slice(tools, func(i, j int) bool {
-		return tools[i].ImportPath < tools[j].ImportPath
-	})
+// InstallSet represents a set of tools that are to be installed.
+// To perform the installation call the Apply method.
+// To abort the install, simply discard the InstallSet object.
+type InstallSet struct {
+	s        *Shed
+	tools    []tool.Tool
+	notifyCh chan<- tool.Tool
+}
 
-	for _, t := range tools {
-		s.logger.Debugf("Installing tool: %v", t)
-		installedTool, err := s.cache.Install(t)
-		if err != nil {
-			return errors.WithMessagef(err, "failed to install tool %s", t)
+// Len returns the number of tools in the InstallSet.
+func (is *InstallSet) Len() int {
+	return len(is.tools)
+}
+
+// Notify causes the InstallSet to relay completed actions to ch.
+// This is useful to keep track of the progress of installation.
+// You should receive from ch on a separate goroutine than the one that
+// Apply is called on, since Apply will block until all tools are installed.
+func (is *InstallSet) Notify(ch chan<- tool.Tool) {
+	is.notifyCh = ch
+}
+
+// Apply will install each tool in the InstallSet and add them to the lockfile.
+//
+// The provided context is used to terminate the install if the context becomes
+// done before the install completes on its own.
+func (is *InstallSet) Apply(ctx context.Context) error {
+	successCh := make(chan tool.Tool)
+	failedCh := make(chan error)
+	for _, tl := range is.tools {
+		go func(t tool.Tool) {
+			is.s.logger.Debugf("Installing tool: %v", t)
+			installed, err := is.s.cache.Install(ctx, t)
+			if err != nil {
+				failedCh <- errors.WithMessagef(err, "failed to install tool %s", t)
+				return
+			}
+			successCh <- installed
+		}(tl)
+	}
+
+	var installedTools []tool.Tool
+	var errs lockfile.ErrorList
+	for i := 0; i < len(is.tools); i++ {
+		select {
+		case t := <-successCh:
+			installedTools = append(installedTools, t)
+			if is.notifyCh != nil {
+				is.notifyCh <- t
+			}
+		case err := <-failedCh:
+			// Continue even if a tool failed because they are cached so it will
+			// save work on subsequent runs.
+			errs = append(errs, err)
+		case <-ctx.Done():
+			return errors.Wrap(ctx.Err(), "installation was aborted")
 		}
-		s.lf.PutTool(installedTool)
+	}
+	if len(errs) > 0 {
+		return errs
 	}
 
-	if err := s.writeLockfile(); err != nil {
+	for _, t := range installedTools {
+		is.s.lf.PutTool(t)
+	}
+	if err := is.s.writeLockfile(); err != nil {
 		return err
 	}
 	return nil

client/client_test.go

@@ -1,6 +1,7 @@
 package client_test
 
 import (
+	"context"
 	"errors"
 	"os"
 	"path/filepath"
@@ -94,6 +95,7 @@ func TestInstall(t *testing.T) {
 		lockfileTools []tool.Tool
 		installTools  []string
 		allowUpdates  bool
+		wantLen       int
 		wantTools     []tool.Tool
 	}{
 		{
@@ -105,6 +107,7 @@ func TestInstall(t *testing.T) {
 				"github.com/Shopify/ejson/cmd/ejson",
 			},
 			allowUpdates: false,
+			wantLen:      3,
 			wantTools: []tool.Tool{
 				{ImportPath: "github.com/cszatmary/go-fish", Version: "v0.1.0"},
 				{ImportPath: "github.com/golangci/golangci-lint/cmd/golangci-lint", Version: "v1.33.0"},
@@ -120,6 +123,7 @@ func TestInstall(t *testing.T) {
 				"github.com/Shopify/ejson/cmd/ejson@v1.1.0",
 			},
 			allowUpdates: false,
+			wantLen:      3,
 			wantTools: []tool.Tool{
 				{ImportPath: "github.com/cszatmary/go-fish", Version: "v0.0.0-20201203230243-22d10c9b658d"},
 				{ImportPath: "github.com/golangci/golangci-lint/cmd/golangci-lint", Version: "v1.28.3"},
@@ -135,6 +139,7 @@ func TestInstall(t *testing.T) {
 			},
 			installTools: nil,
 			allowUpdates: false,
+			wantLen:      3,
 			wantTools: []tool.Tool{
 				{ImportPath: "github.com/cszatmary/go-fish", Version: "v0.1.0"},
 				{ImportPath: "github.com/golangci/golangci-lint/cmd/golangci-lint", Version: "v1.28.3"},
@@ -152,6 +157,7 @@ func TestInstall(t *testing.T) {
 				"github.com/golangci/golangci-lint/cmd/golangci-lint@v1.33.0",
 			},
 			allowUpdates: true,
+			wantLen:      3,
 			wantTools: []tool.Tool{
 				{ImportPath: "github.com/cszatmary/go-fish", Version: "v0.1.0"},
 				{ImportPath: "github.com/golangci/golangci-lint/cmd/golangci-lint", Version: "v1.33.0"},
@@ -182,7 +188,14 @@ func TestInstall(t *testing.T) {
 				t.Fatalf("failed to create shed client %v", err)
 			}
 
-			err = s.Install(tt.allowUpdates, tt.installTools...)
+			installSet, err := s.Install(tt.allowUpdates, tt.installTools...)
+			if err != nil {
+				t.Errorf("want nil error, got %v", err)
+			}
+			if installSet.Len() != tt.wantLen {
+				t.Errorf("want install set len %d, got %d", tt.wantLen, installSet.Len())
+			}
+			err = installSet.Apply(context.Background())
 			if err != nil {
 				t.Errorf("want nil error, got %v", err)
 			}
@@ -225,7 +238,7 @@ func TestInstallError(t *testing.T) {
 		t.Fatalf("failed to create shed client %v", err)
 	}
 
-	err = s.Install(
+	_, err = s.Install(
 		false,
 		"github.com/cszatmary/go-fish",
 		"golangci-lint",

cmd/cache.go

@@ -21,6 +21,7 @@ var cacheCleanCmd = &cobra.Command{
 	Long: `Cleans the shed cache by removing all installed tools.
 This is useful for removing any stale tools that are no longer needed.`,
 	Run: func(cmd *cobra.Command, args []string) {
+		shed := mustShed()
 		if err := shed.CleanCache(); err != nil {
 			fatal.ExitErrf(err, "Failed to clean cache directory")
 		}
@@ -32,6 +33,7 @@ var cacheDirCmd = &cobra.Command{
 	Short: "Prints the path to the shed cache directory.",
 	Long:  `Prints the absolute path to the root shed cache directory where tools are installed.`,
 	Run: func(cmd *cobra.Command, args []string) {
+		shed := mustShed()
 		fmt.Println(shed.CacheDir())
 	},
 }