feat: Add support for module queries

cache/cache.go

@@ -122,7 +122,7 @@ func (c *Cache) download(t tool.Tool) (tool.Tool, error) {
 	modfilePath := filepath.Join(modDir, "go.mod")
 
 	// If we have the version the process is pretty easy
-	if t.Version != "" {
+	if t.HasSemver() {
 		if util.FileOrDirExists(modfilePath) {
 			// If go.mod already exists, make sure there's no issues with it
 			data, err := ioutil.ReadFile(modfilePath)
@@ -177,11 +177,7 @@ func (c *Cache) download(t tool.Tool) (tool.Tool, error) {
 			}
 		}
 
-		// It's easier to just mkdir -p right now instead of
-		// check if the dir exists beforehand
-		// We can improve this later if needed
-		err = os.MkdirAll(modDir, 0o755)
-		if err != nil {
+		if err := os.MkdirAll(modDir, 0o755); err != nil {
 			return t, errors.Wrapf(err, "failed to create directory %q", modDir)
 		}
 
@@ -209,9 +205,8 @@ func (c *Cache) download(t tool.Tool) (tool.Tool, error) {
 	}
 
 	// Don't have the version, this process is a bit more complicated because
-	// we need to figure out what the latest version is
+	// we need to resolve the correct version.
 
-	// Use import path without version and download latest version
 	if err := os.MkdirAll(modDir, 0o755); err != nil {
 		return t, errors.Wrapf(err, "failed to create directory %q", modDir)
 	}
@@ -233,7 +228,7 @@ func (c *Cache) download(t tool.Tool) (tool.Tool, error) {
 	}
 
 	// Download the module source. This will do the heavy lifting to figure out
-	// the latest version.
+	// the correct version.
 	err = c.goClient.GetD(t.Module(), modDir)
 	if err != nil {
 		return t, err

cache/go.go

@@ -105,30 +105,31 @@ type mockModule struct {
 	name string
 	// List of semver versions, must be sorted from earliest to latest version
 	versions []string
+	// Queries to versions
+	queries map[string]string
 }
 
 // NewMockGo returns a new Go instance that is suitable for testing.
-// Tools is a list of tools which contains the import path and version.
-func NewMockGo(tools []string) (Go, error) {
+// Tools is a map of import paths to a map of queries to versions.
+func NewMockGo(tools map[string]map[string]string) (Go, error) {
 	registry := make(map[string]mockModule)
-	for _, tn := range tools {
-		t, err := tool.Parse(tn)
+	for tn, queries := range tools {
+		t, err := tool.ParseLax(tn)
 		if err != nil {
 			return nil, err
 		}
 
-		// Module already exists, add new version
-		m, ok := registry[t.ImportPath]
-		if ok {
-			m.versions = append(m.versions, t.Version)
-			registry[t.ImportPath] = m
-			continue
-		}
-
 		// Take first 3 parts as the module name
 		// This should be could enough for testing purposes
 		modName := strings.Join(strings.Split(t.ImportPath, "/")[:3], "/")
-		m = mockModule{name: modName, versions: []string{t.Version}}
+		m := mockModule{name: modName, queries: queries}
+		var versions []string
+		for q := range queries {
+			if semver.IsValid(q) && q == semver.Canonical(q) {
+				versions = append(versions, q)
+			}
+		}
+		m.versions = versions
 		registry[t.ImportPath] = m
 	}
 
@@ -157,7 +158,7 @@ func (mg *mockGo) Build(pkg, outPath, dir string) error {
 }
 
 func (mg *mockGo) GetD(mod, dir string) error {
-	t, err := tool.Parse(mod)
+	t, err := tool.ParseLax(mod)
 	if err != nil {
 		return err
 	}
@@ -168,22 +169,33 @@ func (mg *mockGo) GetD(mod, dir string) error {
 	}
 
 	modver := module.Version{Path: m.name}
-	if t.Version != "" {
+	if t.Version == "" || t.Version == "latest" {
+		modver.Version = m.versions[len(m.versions)-1]
+	} else {
 		// If version is provided, see if it exists
 		found := false
-		for _, v := range m.versions {
-			if v == t.Version {
-				modver.Version = v
-				found = true
-				break
+		// TODO(@cszatmary): Make this work with shorthand semvers
+		if t.HasSemver() {
+			for _, v := range m.versions {
+				if v == t.Version {
+					modver.Version = v
+					found = true
+					break
+				}
+			}
+		} else {
+			// If no semver, see if a matching query exists
+			for q, v := range m.queries {
+				if q == t.Version {
+					modver.Version = v
+					found = true
+					break
+				}
 			}
 		}
 		if !found {
 			return errors.Errorf("module %s has no version %s", t.ImportPath, t.Version)
 		}
-	} else {
-		// Find latest version
-		modver.Version = m.versions[len(m.versions)-1]
 	}
 
 	modfilePath := filepath.Join(dir, "go.mod")

client/client.go

@@ -132,9 +132,11 @@ func (s *Shed) Install(allowUpdates bool, toolNames ...string) error {
 	var errs lockfile.ErrorList
 	for _, toolName := range toolNames {
 		// This also serves to validate the the given tool name is a valid module name
-		t, err := tool.Parse(toolName)
+		// Use ParseLax since the version might be a query that should be passed to go get.
+		t, err := tool.ParseLax(toolName)
 		if err != nil {
 			errs = append(errs, errors.WithMessagef(err, "invalid tool name %s", toolName))
+			continue
 		}
 
 		existing, err := s.lf.GetTool(toolName)

client/client_test.go

@@ -38,14 +38,25 @@ func TestClientCache(t *testing.T) {
 	}
 }
 
-var availableTools = []string{
-	"github.com/cszatmary/go-fish@v0.1.0",
-	"github.com/golangci/golangci-lint/cmd/golangci-lint@v1.33.0",
-	"github.com/golangci/golangci-lint/cmd/golangci-lint@v1.28.3",
-	"golang.org/x/tools/cmd/stringer@v0.0.0-20201211185031-d93e913c1a58",
-	"github.com/Shopify/ejson/cmd/ejson@v1.2.2",
-	"github.com/Shopify/ejson/cmd/ejson@v1.1.0",
-	"example.org/z/random/stringer/v2/cmd/stringer@v2.1.0",
+var availableTools = map[string]map[string]string{
+	"github.com/cszatmary/go-fish": {
+		"v0.1.0": "v0.1.0",
+		"22d10c9b658df297b17b33c836a60fb943ef5a5f": "v0.0.0-20201203230243-22d10c9b658d",
+	},
+	"github.com/golangci/golangci-lint/cmd/golangci-lint": {
+		"v1.33.0": "v1.33.0",
+		"v1.28.3": "v1.28.3",
+	},
+	"golang.org/x/tools/cmd/stringer": {
+		"v0.0.0-20201211185031-d93e913c1a58": "v0.0.0-20201211185031-d93e913c1a58",
+	},
+	"github.com/Shopify/ejson/cmd/ejson": {
+		"v1.2.2": "v1.2.2",
+		"v1.1.0": "v1.1.0",
+	},
+	"example.org/z/random/stringer/v2/cmd/stringer": {
+		"v2.1.0": "v2.1.0",
+	},
 }
 
 func createLockfile(t *testing.T, path string, tools []tool.Tool) {
@@ -106,13 +117,13 @@ func TestInstall(t *testing.T) {
 			name:          "install specific versions",
 			lockfileTools: nil,
 			installTools: []string{
-				"github.com/cszatmary/go-fish@v0.1.0",
+				"github.com/cszatmary/go-fish@22d10c9b658df297b17b33c836a60fb943ef5a5f",
 				"github.com/golangci/golangci-lint/cmd/golangci-lint@v1.28.3",
 				"github.com/Shopify/ejson/cmd/ejson@v1.1.0",
 			},
 			allowUpdates: false,
 			wantTools: []tool.Tool{
-				{ImportPath: "github.com/cszatmary/go-fish", Version: "v0.1.0"},
+				{ImportPath: "github.com/cszatmary/go-fish", Version: "v0.0.0-20201203230243-22d10c9b658d"},
 				{ImportPath: "github.com/golangci/golangci-lint/cmd/golangci-lint", Version: "v1.28.3"},
 				{ImportPath: "github.com/Shopify/ejson/cmd/ejson", Version: "v1.1.0"},
 			},

lockfile/lockfile.go

@@ -24,7 +24,7 @@ var ErrIncorrectVersion = errors.New("lockfile: incorrect version of tool")
 var ErrMultipleTools = errors.New("lockfile: multiple tools found with the same name")
 
 // Lockfile represents a shed lockfile. The lockfile is responsible for keeping
-// track of installed tools as well as their versions should shed can always
+// track of installed tools as well as their versions so shed can always
 // re-install the same version of each tool.
 //
 // An a zero value Lockfile is a valid empty lockfile ready for use.
@@ -66,7 +66,7 @@ func (lf *Lockfile) GetTool(name string) (tool.Tool, error) {
 	}
 
 	// Long way, parse the tool name which should be an import path
-	tl, err := tool.Parse(name)
+	tl, err := tool.ParseLax(name)
 	if err != nil {
 		return tool.Tool{}, err
 	}
@@ -92,7 +92,7 @@ func (lf *Lockfile) GetTool(name string) (tool.Tool, error) {
 	return tool.Tool{}, fmt.Errorf("%w: %s", ErrNotFound, toolName)
 }
 
-// PutTool add or replaces the given tool in the lockfile.
+// PutTool adds or replaces the given tool in the lockfile.
 func (lf *Lockfile) PutTool(t tool.Tool) {
 	if lf.tools == nil {
 		lf.tools = make(map[string][]tool.Tool)

lockfile/lockfile_test.go

@@ -81,6 +81,13 @@ func TestLockfileGet(t *testing.T) {
 			wantTool: tool.Tool{ImportPath: "golang.org/x/tools/cmd/stringer", Version: "v0.0.0-20201211185031-d93e913c1a58"},
 			wantErr:  lockfile.ErrIncorrectVersion,
 		},
+		{
+			// Make sure it is not found instead of invalid version
+			name:     "not found query",
+			toolName: "golang.org/x/tools/cmd/stress@master",
+			wantTool: tool.Tool{},
+			wantErr:  lockfile.ErrNotFound,
+		},
 	}
 
 	for _, tt := range tests {

tool/tool.go

@@ -3,11 +3,11 @@
 package tool
 
 import (
+	"fmt"
 	"path"
 	"path/filepath"
 	"strings"
 
-	"github.com/pkg/errors"
 	"golang.org/x/mod/module"
 	"golang.org/x/mod/semver"
 )
@@ -43,6 +43,14 @@ func (t Tool) Module() string {
 	return t.ImportPath + "@" + t.Version
 }
 
+// HasSemver reports whether t.Version is a valid semantic version.
+// HasSemver requires t.Version to be a full semantic version. It does
+// not allow shorthands like vMAJOR or vMAJOR.MINOR.
+func (t Tool) HasSemver() bool {
+	// Compare against canonical to make sure it isn't a shorthand.
+	return semver.IsValid(t.Version) && t.Version == semver.Canonical(t.Version)
+}
+
 // String returns a string representation of the tool.
 func (t Tool) String() string {
 	// While this may seem shallow, String serves a different purpose
@@ -59,13 +67,13 @@ func (t Tool) String() string {
 func (t Tool) Filepath() (string, error) {
 	escapedPath, err := module.EscapePath(t.ImportPath)
 	if err != nil {
-		return "", errors.Wrapf(err, "tool: failed to escape path %q", t.ImportPath)
+		return "", fmt.Errorf("tool: failed to escape path %q: %w", t.ImportPath, err)
 	}
 
 	if t.Version != "" {
 		escapedVersion, err := module.EscapeVersion(t.Version)
 		if err != nil {
-			return "", errors.Wrapf(err, "tool: failed to escape version %q", t.Version)
+			return "", fmt.Errorf("tool: failed to escape version %q: %w", t.Version, err)
 		}
 		escapedPath += "@" + escapedVersion
 	}
@@ -78,31 +86,67 @@ func (t Tool) Filepath() (string, error) {
 func (t Tool) BinaryFilepath() (string, error) {
 	fp, err := t.Filepath()
 	if err != nil {
-		return "", errors.WithMessage(err, "tool: failed to get filepath")
+		return "", err
 	}
 	return filepath.Join(fp, t.Name()), nil
 }
 
-// Parse parses the given tool name. Name must be a valid import path,
-// optionally with a version. If a version is provided, the format must be
-// 'ImportPath@Version', just like what would be passed to a command like 'go get'.
+// Parse parses the given tool name and returns a tool containing the
+// import path and version. name must be a valid import path and a version
+// with the format 'IMPORT_PATH@VERSION'. This format is the same as what would be
+// pass to a command like 'go get'. The version must be a valid semantic version
+// and it must be prefixed with 'v' (ex: 'v1.2.3'). If a shorthand semantic version
+// is used, it will be canonicalized (ex: 'v1' will become 'v1.0.0').
 func Parse(name string) (Tool, error) {
+	return parseTool(name, true)
+}
+
+// ParseLax is like Parse but does not check that the version is a valid semantic version.
+// It is used when downloading and resolving tools using 'go get'. This is because
+// go get allows module queries, which is where a version is resolved based on a
+// branch name, commit SHA, version range, etc.
+// See https://golang.org/cmd/go/#hdr-Module_queries for more details on module queries.
+// Unlike Parse, ParseLax will not canonicalize shorthand semantic verions and will
+// instead leave them as is.
+//
+// ParseLax allows the version to be omitted in which case it is assumed to mean
+// the latest version. That is, 'golang/x/tools/cmd/stringer' is functionally
+// equivalent to 'golang/x/tools/cmd/stringer@latest'.
+func ParseLax(name string) (Tool, error) {
+	return parseTool(name, false)
+}
+
+func parseTool(name string, strict bool) (Tool, error) {
 	t := Tool{ImportPath: name}
 
-	// Check if version is provided
+	// Check if a version/query is provided
 	if i := strings.IndexByte(name, '@'); i != -1 {
 		t.ImportPath = name[:i]
 		t.Version = name[i+1:]
+
+		// Make sure there isn't a dangling '@'
+		if t.Version == "" {
+			return t, fmt.Errorf("tool: missing version after '@'")
+		}
 	}
 
 	// Validations
 	if err := module.CheckPath(t.ImportPath); err != nil {
-		return t, errors.Wrapf(err, "tool: invalid import path: %q", t.ImportPath)
+		return t, fmt.Errorf("tool: invalid import path %q: %w", t.ImportPath, err)
 	}
-
-	if t.Version != "" && !semver.IsValid(t.Version) {
-		return t, errors.Errorf("tool: invalid version: %q", t.Version)
+	// Version validation is ignored if not strict
+	if !strict {
+		return t, nil
 	}
 
+	if !semver.IsValid(t.Version) {
+		return t, fmt.Errorf("tool: invalid version %q: not a semantic version", t.Version)
+	}
+	// The semver package allows vMAJOR and vMAJOR.MINOR as shorthands.
+	// Use the canonical version to ensure it is a full semantic version.
+	canonical := semver.Canonical(t.Version)
+	if t.Version != canonical {
+		t.Version = canonical
+	}
 	return t, nil
 }