From d167d92f22af1e2969a6d7490ba108258b4bdfbb Mon Sep 17 00:00:00 2001 From: Mattias Wadman Date: Tue, 18 Oct 2022 19:36:28 +0200 Subject: [PATCH] docker: Add registry support --- README.md | 25 +++-- internal/dockerv2/dockerv2.go | 160 +++++++++++++++++++++++++++++ internal/dockerv2/dockerv2_test.go | 20 ++++ internal/filter/docker/docker.go | 91 +++------------- internal/filter/git/git.go | 1 - internal/pipeline/testdata/docker | 8 ++ 6 files changed, 217 insertions(+), 88 deletions(-) create mode 100644 internal/dockerv2/dockerv2.go create mode 100644 internal/dockerv2/dockerv2_test.go diff --git a/README.md b/README.md index 32a1451..1259c05 100644 --- a/README.md +++ b/README.md @@ -20,14 +20,14 @@ $ bump current Dockerfile:1: alpine 3.9.2 # See possible updates $ bump check -alpine 3.14.2 +alpine 3.16.2 # See what will be changed $ bump diff --- Dockerfile +++ Dockerfile @@ -1,2 +1,2 @@ -FROM alpine:3.9.2 AS builder -+FROM alpine:3.14.2 AS builder ++FROM alpine:3.16.2 AS builder # Write changes $ bump update @@ -275,13 +275,13 @@ can be helpful when testing pipelines. ```sh (exec) # Latest 4.0 ffmpeg version $ bump pipeline 'https://github.com/FFmpeg/FFmpeg.git|^4' -4.4.1 +4.4.3 # Commit hash of the latest 4.0 ffmpeg version $ bump pipeline 'https://github.com/FFmpeg/FFmpeg.git|^4|@commit' -2aa4f5cc8be5e3168191cd13a61178a167687eac +3d69f9682f06bbf72e0cdcdc9e66c9307ed6b24f # Latest 1.0 golang docker build image $ bump pipeline 'docker:golang|^1' -1.17.3 +1.19.2 # Latest mp3lame version $ bump pipeline 'svn:http://svn.code.sf.net/p/lame/svn|/^RELEASE__(.*)$/|/_/./|*' 3.100 @@ -316,9 +316,7 @@ Use gitrefs filter to get all refs unfiltered. ```sh $ bump pipeline 'https://github.com/git/git.git|*' -2.33.1 -$ bump pipeline 'git://github.com/git/git.git|*' -2.33.1 +2.38.1 ``` ### gitrefs @@ -339,11 +337,16 @@ HEAD `docker:` -Produce versions from a image on ducker hub. +Produce versions from a image on ducker hub or other registry. +Currently only supports anonymous access. ```sh $ bump pipeline 'docker:alpine|^3' -3.14.2 +3.16.2 +$ bump pipeline 'docker:mwader/static-ffmpeg|^4' +4.4.1 +$ bump pipeline 'docker:ghcr.io/nginx-proxy/nginx-proxy|^0.9' +0.9.3 ``` ### svn @@ -355,7 +358,7 @@ be the tag or branch name, version the revision. ```sh $ bump pipeline 'svn:https://svn.apache.org/repos/asf/subversion|*' -1.14.1 +1.14.2 ``` ### fetch diff --git a/internal/dockerv2/dockerv2.go b/internal/dockerv2/dockerv2.go new file mode 100644 index 0000000..d843df5 --- /dev/null +++ b/internal/dockerv2/dockerv2.go @@ -0,0 +1,160 @@ +package dockerv2 + +import ( + "encoding/csv" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" +) + +type Registry struct { + Host string + Image string + Token string +} + +var defaultRegistry = Registry{ + Host: "index.docker.io", +} + +const listTagsURLTemplate = `https://%s/v2/%s/tags/list` + +func NewFromImage(image string) (*Registry, error) { + parts := strings.Split(image, "/") + r := defaultRegistry + switch { + case len(parts) == 0: + return &r, fmt.Errorf("invalid image") + case len(parts) == 1: + // image + r.Image = "library/" + image + return &r, nil + default: + } + + if strings.Contains(parts[0], ".") { + // host.tldr/image + r.Host = parts[0] + r.Image = strings.Join(parts[1:], "/") + return &r, nil + } + + // repo/image + r.Image = image + return &r, nil +} + +// The WWW-Authenticate Response Header Field +// https://www.rfc-editor.org/rfc/rfc6750#section-3 +type WWWAuth struct { + Scheme string + Params map[string]string +} + +// WWW-Authenticate: Bearer realm="https://ghcr.io/token",service="ghcr.io",scope="repository:org/image:pull"" +func ParseWWWAuth(s string) (WWWAuth, error) { + var w WWWAuth + parts := strings.SplitN(s, " ", 2) + if len(parts) != 2 { + return WWWAuth{}, fmt.Errorf("invalid params") + } + w.Scheme = parts[0] + + r := csv.NewReader(strings.NewReader(strings.TrimSpace(parts[1]))) + // allows mix quotes and explicit "," + r.LazyQuotes = true + r.Comma = rune(',') + pairs, pairsErr := r.Read() + if pairsErr != nil { + return WWWAuth{}, pairsErr + } + + w.Params = map[string]string{} + for _, p := range pairs { + r := csv.NewReader(strings.NewReader(p)) + r.Comma = rune('=') + kv, kvErr := r.Read() + if kvErr != nil { + return WWWAuth{}, kvErr + } + if len(kv) != 2 { + return WWWAuth{}, fmt.Errorf("invalid pair") + } + w.Params[kv[0]] = kv[1] + } + + return w, nil +} + +func get(rawURL string, doAuth bool, token string, out interface{}) error { + req, err := http.NewRequest(http.MethodGet, rawURL, nil) + if err != nil { + return err + } + + if token != "" { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + } + + r, err := http.DefaultClient.Do(req) + if err != nil { + return fmt.Errorf("request failed: %w", err) + } + defer r.Body.Close() + + if r.StatusCode/100 == 4 { + if doAuth && r.StatusCode == 401 { + wwwAuth := r.Header.Get("WWW-Authenticate") + if wwwAuth == "" { + return fmt.Errorf("no WWW-Authenticate found") + } + + w, wwwAuthErr := ParseWWWAuth(wwwAuth) + if wwwAuthErr != nil { + return wwwAuthErr + } + + authURLValues := url.Values{} + authURLValues.Set("service", w.Params["service"]) + authURLValues.Set("scope", w.Params["scope"]) + authURL, authURLErr := url.Parse(w.Params["realm"]) + if authURLErr != nil { + return authURLErr + } + authURL.RawQuery = authURLValues.Encode() + + var authResp struct { + Token string `json:"token"` + } + authTokenErr := get(authURL.String(), false, "", &authResp) + if authTokenErr != nil { + return authTokenErr + } + + return get(rawURL, false, authResp.Token, out) + } + return fmt.Errorf(r.Status) + } + + if r.StatusCode/100 != 2 { + return fmt.Errorf("error response: %s", r.Status) + } + + if err := json.NewDecoder(r.Body).Decode(&out); err != nil { + return fmt.Errorf("failed parse response: %w", err) + } + + return nil +} + +func (r *Registry) Tags() ([]string, error) { + var resp struct { + Tags []string `json:"tags"` + } + if err := get(fmt.Sprintf(listTagsURLTemplate, r.Host, r.Image), true, "", &resp); err != nil { + return nil, err + } + return resp.Tags, nil +} diff --git a/internal/dockerv2/dockerv2_test.go b/internal/dockerv2/dockerv2_test.go new file mode 100644 index 0000000..3eac94f --- /dev/null +++ b/internal/dockerv2/dockerv2_test.go @@ -0,0 +1,20 @@ +package dockerv2_test + +import ( + "testing" + + "github.com/wader/bump/internal/dockerv2" +) + +func TestParseWWWAuth(t *testing.T) { + w, err := dockerv2.ParseWWWAuth(`Bearer realm="https://ghcr.io/token",service="ghcr.io",scope="repository:org/image:pull"`) + if err != nil { + t.Fatal(err) + } + if w.Scheme != "Bearer" { + t.Fatalf("schema %s", w.Scheme) + } + if v := w.Params["service"]; v != "ghcr.io" { + t.Fatalf("service %s", v) + } +} diff --git a/internal/filter/docker/docker.go b/internal/filter/docker/docker.go index 6f5c2a7..2065b36 100644 --- a/internal/filter/docker/docker.go +++ b/internal/filter/docker/docker.go @@ -1,11 +1,9 @@ package docker import ( - "encoding/json" "fmt" - "net/http" - "strings" + "github.com/wader/bump/internal/dockerv2" "github.com/wader/bump/internal/filter" ) @@ -16,30 +14,14 @@ const Name = "docker" var Help = ` docker: -Produce versions from a image on ducker hub. +Produce versions from a image on ducker hub or other registry. +Currently only supports anonymous access. docker:alpine|^3 +docker:mwader/static-ffmpeg|^4 +docker:ghcr.io/nginx-proxy/nginx-proxy|^0.9 `[1:] -// TODO: support other registries -// TODO: auth? -const authURLTemplate = `https://auth.docker.io/token?service=registry.docker.io&scope=repository:%s:pull` -const listTagsURLTemplate = `https://index.docker.io/v2/%s/tags/list` - -// image -> library/image -// repo/image -> repo/image -func argToRepoImage(a string) (string, error) { - parts := strings.Split(a, "/") - switch { - case len(parts) == 0: - return "", fmt.Errorf("invalid name") - case len(parts) == 1: - return "library/" + parts[0], nil - default: - return a, nil - } -} - // New docker filter func New(prefix string, arg string) (filter filter.Filter, err error) { if prefix != Name { @@ -49,77 +31,34 @@ func New(prefix string, arg string) (filter filter.Filter, err error) { return nil, fmt.Errorf("needs a image name") } - repoImage, err := argToRepoImage(arg) + registry, err := dockerv2.NewFromImage(arg) if err != nil { return nil, fmt.Errorf("%w: %s", err, arg) } return dockerFilter{ - imageName: arg, - repoImage: repoImage, + image: arg, + registry: registry, }, nil } type dockerFilter struct { - imageName string - repoImage string + image string + registry *dockerv2.Registry } func (f dockerFilter) String() string { - return Name + ":" + f.imageName -} - -func (f dockerFilter) getToken() (string, error) { - r, err := http.Get(fmt.Sprintf(authURLTemplate, f.repoImage)) - if err != nil { - return "", fmt.Errorf("token request failed: %w", err) - } - defer r.Body.Close() - - var resp struct { - Token string `json:"token"` - } - - if err := json.NewDecoder(r.Body).Decode(&resp); err != nil { - return "", fmt.Errorf("failed to parse token response: %w", err) - } - - return resp.Token, nil + return Name + ":" + f.image } func (f dockerFilter) Filter(versions filter.Versions, versionKey string) (filter.Versions, string, error) { - token, err := f.getToken() - if err != nil { - return nil, "", err - } - - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(listTagsURLTemplate, f.repoImage), nil) - if err != nil { - return nil, "", err - } - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) - - r, err := http.DefaultClient.Do(req) - if err != nil { - return nil, versionKey, err - } - defer r.Body.Close() - - if r.StatusCode/100 != 2 { - return nil, "", fmt.Errorf("error response: %s", r.Status) - } - - var resp struct { - Tags []string `json:"tags"` - } - - err = json.NewDecoder(r.Body).Decode(&resp) - if err != nil { - return nil, "", err + tags, tagsErr := f.registry.Tags() + if tagsErr != nil { + return filter.Versions{}, "", tagsErr } tagNames := append(filter.Versions{}, versions...) - for _, t := range resp.Tags { + for _, t := range tags { tagNames = append(tagNames, filter.NewVersionWithName(t, nil)) } diff --git a/internal/filter/git/git.go b/internal/filter/git/git.go index 2f7e547..57aa99d 100644 --- a/internal/filter/git/git.go +++ b/internal/filter/git/git.go @@ -22,7 +22,6 @@ the version found in the tag, commit the commit hash or tag object. Use gitrefs filter to get all refs unfiltered. https://github.com/git/git.git|* -git://github.com/git/git.git|* `[1:] // default ref filter diff --git a/internal/pipeline/testdata/docker b/internal/pipeline/testdata/docker index 703a7a6..60bea24 100644 --- a/internal/pipeline/testdata/docker +++ b/internal/pipeline/testdata/docker @@ -1,2 +1,10 @@ docker:alpine|^2 -> docker:alpine|semver:^2 -> 2.7 2.7 +docker:mwader/static-ffmpeg|^4 -> docker:mwader/static-ffmpeg|semver:^4 + -> 4.4.1 4.4.1 +docker:gcr.io/google.com/cloudsdktool/google-cloud-cli|=365.0.1-alpine -> docker:gcr.io/google.com/cloudsdktool/google-cloud-cli|semver:=365.0.1-alpine + -> 365.0.1-alpine 365.0.1-alpine +docker:ghcr.io/nginx-proxy/nginx-proxy|^0.9 -> docker:ghcr.io/nginx-proxy/nginx-proxy|semver:^0.9 + -> 0.9.3 0.9.3 +docker:non/existing -> docker:non/existing + -> error:401 Unauthorized