From 122b3deb94bb963ccfbf1591a837394251ebaa65 Mon Sep 17 00:00:00 2001 From: k124k3n Date: Mon, 15 Apr 2024 20:03:27 +0700 Subject: [PATCH 01/14] Upgrade to AWS SDK v2 (#1938) --- pkg/storage/s3/cataloger.go | 11 +++-- pkg/storage/s3/checker.go | 8 +-- pkg/storage/s3/deleter.go | 8 +-- pkg/storage/s3/getter.go | 24 ++++----- pkg/storage/s3/lister.go | 9 ++-- pkg/storage/s3/s3.go | 98 ++++++++++++++++++++++--------------- pkg/storage/s3/s3_test.go | 52 ++++++++++++-------- pkg/storage/s3/saver.go | 9 ++-- 8 files changed, 127 insertions(+), 92 deletions(-) diff --git a/pkg/storage/s3/cataloger.go b/pkg/storage/s3/cataloger.go index 193085718..f1e591042 100644 --- a/pkg/storage/s3/cataloger.go +++ b/pkg/storage/s3/cataloger.go @@ -3,10 +3,11 @@ package s3 import ( "context" "fmt" + "github.com/aws/aws-sdk-go-v2/service/s3/types" "strings" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/observ" @@ -28,7 +29,7 @@ func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]pa Marker: &queryToken, } - loo, err := s.s3API.ListObjectsWithContext(ctx, lsParams) + loo, err := s.s3API.ListObjects(ctx, lsParams) if err != nil { return nil, "", errors.E(op, err) } @@ -49,7 +50,7 @@ func (s *Storage) Catalog(ctx context.Context, token string, pageSize int) ([]pa return res, queryToken, nil } -func fetchModsAndVersions(objects []*s3.Object, elementsNum int) ([]paths.AllPathParams, string) { +func fetchModsAndVersions(objects []types.Object, elementsNum int) ([]paths.AllPathParams, string) { res := make([]paths.AllPathParams, 0) lastKey := "" for _, o := range objects { @@ -72,7 +73,7 @@ func fetchModsAndVersions(objects []*s3.Object, elementsNum int) ([]paths.AllPat return res, lastKey } -func parseS3Key(o *s3.Object) (paths.AllPathParams, error) { +func parseS3Key(o types.Object) (paths.AllPathParams, error) { const op errors.Op = "s3.parseS3Key" m, v := config.ModuleVersionFromPath(*o.Key) diff --git a/pkg/storage/s3/checker.go b/pkg/storage/s3/checker.go index b5d039c10..b447c6dcd 100644 --- a/pkg/storage/s3/checker.go +++ b/pkg/storage/s3/checker.go @@ -2,11 +2,11 @@ package s3 import ( "context" + "github.com/aws/aws-sdk-go/aws/awserr" "sync" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/observ" @@ -28,7 +28,7 @@ func (s *Storage) Exists(ctx context.Context, module, version string) (bool, err wg.Add(1) go func(file string) { defer wg.Done() - _, err := s.s3API.HeadObjectWithContext( + _, err := s.s3API.HeadObject( cancelingCtx, &s3.HeadObjectInput{ Bucket: aws.String(s.bucket), diff --git a/pkg/storage/s3/deleter.go b/pkg/storage/s3/deleter.go index c2589e9f1..4651cefed 100644 --- a/pkg/storage/s3/deleter.go +++ b/pkg/storage/s3/deleter.go @@ -2,9 +2,8 @@ package s3 import ( "context" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/observ" modupl "github.com/gomods/athens/pkg/storage/module" @@ -32,12 +31,13 @@ func (s *Storage) remove(ctx context.Context, path string) error { const op errors.Op = "s3.Delete" ctx, span := observ.StartSpan(ctx, op.String()) defer span.End() + delParams := &s3.DeleteObjectInput{ Bucket: aws.String(s.bucket), Key: aws.String(path), } - if _, err := s.s3API.DeleteObjectWithContext(ctx, delParams); err != nil { + if _, err := s.s3API.DeleteObject(ctx, delParams); err != nil { return errors.E(op, err) } diff --git a/pkg/storage/s3/getter.go b/pkg/storage/s3/getter.go index f3c748dcf..ab01b7e98 100644 --- a/pkg/storage/s3/getter.go +++ b/pkg/storage/s3/getter.go @@ -3,11 +3,11 @@ package s3 import ( "context" "fmt" + "github.com/aws/aws-sdk-go-v2/service/s3/types" "io" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/observ" @@ -22,8 +22,8 @@ func (s *Storage) Info(ctx context.Context, module, version string) ([]byte, err infoReader, err := s.open(ctx, config.PackageVersionedName(module, version, "info")) if err != nil { - var aerr awserr.Error - if errors.AsErr(err, &aerr) && aerr.Code() == s3.ErrCodeNoSuchKey { + var nsk *types.NoSuchKey + if errors.AsErr(err, &nsk) { return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound) } return nil, errors.E(op, err, errors.M(module), errors.V(version)) @@ -45,8 +45,8 @@ func (s *Storage) GoMod(ctx context.Context, module, version string) ([]byte, er modReader, err := s.open(ctx, config.PackageVersionedName(module, version, "mod")) if err != nil { - var aerr awserr.Error - if errors.AsErr(err, &aerr) && aerr.Code() == s3.ErrCodeNoSuchKey { + var nsk *types.NoSuchKey + if errors.AsErr(err, &nsk) { return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound) } return nil, errors.E(op, err, errors.M(module), errors.V(version)) @@ -69,8 +69,8 @@ func (s *Storage) Zip(ctx context.Context, module, version string) (storage.Size zipReader, err := s.open(ctx, config.PackageVersionedName(module, version, "zip")) if err != nil { - var aerr awserr.Error - if errors.AsErr(err, &aerr) && aerr.Code() == s3.ErrCodeNoSuchKey { + var nsk *types.NoSuchKey + if errors.AsErr(err, &nsk) { return nil, errors.E(op, errors.M(module), errors.V(version), errors.KindNotFound) } return nil, errors.E(op, err, errors.M(module), errors.V(version)) @@ -88,10 +88,10 @@ func (s *Storage) open(ctx context.Context, path string) (storage.SizeReadCloser Key: aws.String(path), } - goo, err := s.s3API.GetObjectWithContext(ctx, getParams) + goo, err := s.s3API.GetObject(ctx, getParams) if err != nil { - var aerr awserr.Error - if errors.AsErr(err, &aerr) && aerr.Code() == s3.ErrCodeNoSuchKey { + var nsk *types.NoSuchKey + if errors.AsErr(err, &nsk) { return nil, errors.E(op, errors.KindNotFound) } return nil, errors.E(op, err) diff --git a/pkg/storage/s3/lister.go b/pkg/storage/s3/lister.go index 9b51c6d6c..e593e88ea 100644 --- a/pkg/storage/s3/lister.go +++ b/pkg/storage/s3/lister.go @@ -2,10 +2,11 @@ package s3 import ( "context" + "github.com/aws/aws-sdk-go-v2/service/s3/types" "strings" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/observ" ) @@ -23,7 +24,7 @@ func (s *Storage) List(ctx context.Context, module string) ([]string, error) { Prefix: aws.String(modulePrefix), } - loo, err := s.s3API.ListObjectsWithContext(ctx, lsParams) + loo, err := s.s3API.ListObjects(ctx, lsParams) if err != nil { return nil, errors.E(op, err, errors.M(module)) } @@ -31,7 +32,7 @@ func (s *Storage) List(ctx context.Context, module string) ([]string, error) { return extractVersions(loo.Contents), nil } -func extractVersions(objects []*s3.Object) []string { +func extractVersions(objects []types.Object) []string { var versions []string for _, o := range objects { diff --git a/pkg/storage/s3/s3.go b/pkg/storage/s3/s3.go index 813db57d9..66b1c94bf 100644 --- a/pkg/storage/s3/s3.go +++ b/pkg/storage/s3/s3.go @@ -1,18 +1,17 @@ package s3 import ( - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/credentials/endpointcreds" - "github.com/aws/aws-sdk-go/aws/defaults" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3/s3iface" - "github.com/aws/aws-sdk-go/service/s3/s3manager" - "github.com/aws/aws-sdk-go/service/s3/s3manager/s3manageriface" + "context" + "fmt" + "github.com/aws/aws-sdk-go-v2/aws" + awscfg "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/credentials/endpointcreds" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/errors" + "time" ) // Storage implements (./pkg/storage).Backend and @@ -24,10 +23,16 @@ import ( // - AWS_SESSION_TOKEN - [optional] // - AWS_FORCE_PATH_STYLE - [optional] // For information how to get your keyId and access key turn to official aws docs: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/setting-up.html. + +type s3API interface { + manager.DeleteObjectsAPIClient + manager.DownloadAPIClient + manager.ListObjectsV2APIClient +} type Storage struct { bucket string - uploader s3manageriface.UploaderAPI - s3API s3iface.S3API + uploader *manager.Uploader + s3API *s3.Client timeout time.Duration } @@ -35,50 +40,45 @@ type Storage struct { func New(s3Conf *config.S3Config, timeout time.Duration, options ...func(*aws.Config)) (*Storage, error) { const op errors.Op = "s3.New" - awsConfig := defaults.Config() + awsConfig, err := awscfg.LoadDefaultConfig(context.TODO(), awscfg.WithRegion(s3Conf.Region)) + + if err != nil { + return nil, errors.E(op, err) + } + // Remove anonymous credentials from the default config so that // session.NewSession can auto-resolve credentials from role, profile, env etc. awsConfig.Credentials = nil - awsConfig.Region = aws.String(s3Conf.Region) + for _, o := range options { - o(awsConfig) + o(&awsConfig) } if !s3Conf.UseDefaultConfiguration { - credProviders := defaults.CredProviders(awsConfig, defaults.Handlers()) - endpointcreds := []credentials.Provider{ - endpointcreds.NewProviderClient(*awsConfig, defaults.Handlers(), endpointFrom(s3Conf.CredentialsEndpoint, s3Conf.AwsContainerCredentialsRelativeURI)), - &credentials.StaticProvider{ - Value: credentials.Value{ - AccessKeyID: s3Conf.Key, - SecretAccessKey: s3Conf.Secret, - SessionToken: s3Conf.Token, - }, - }, + //credProviders := defaults.CredProviders(awsConfig, defaults.Handlers()) + endpointCreds := []aws.CredentialsProvider{ + endpointcreds.New(endpointFrom(s3Conf.CredentialsEndpoint, s3Conf.AwsContainerCredentialsRelativeURI)), + credentials.NewStaticCredentialsProvider(s3Conf.Key, s3Conf.Secret, s3Conf.Token), } - credProviders = append(endpointcreds, credProviders...) - awsConfig.Credentials = credentials.NewChainCredentials(credProviders) - } - - awsConfig.S3ForcePathStyle = aws.Bool(s3Conf.ForcePathStyle) - awsConfig.CredentialsChainVerboseErrors = aws.Bool(true) - if s3Conf.Endpoint != "" { - awsConfig.Endpoint = aws.String(s3Conf.Endpoint) + //credProviders = append(endpointCreds, credProviders...) + awsConfig.Credentials = newChainCredentials(endpointCreds...) } // Create a session with creds. - sess, err := session.NewSession(awsConfig) - if err != nil { - return nil, errors.E(op, err) - } + sess := s3.NewFromConfig(awsConfig, func(o *s3.Options) { + o.UsePathStyle = s3Conf.ForcePathStyle + if s3Conf.Endpoint != "" { + o.BaseEndpoint = aws.String(s3Conf.Endpoint) + } + }) - uploader := s3manager.NewUploader(sess) + uploader := manager.NewUploader(sess) return &Storage{ bucket: s3Conf.Bucket, uploader: uploader, - s3API: uploader.S3, + s3API: sess, timeout: timeout, }, nil } @@ -86,3 +86,23 @@ func New(s3Conf *config.S3Config, timeout time.Duration, options ...func(*aws.Co func endpointFrom(credentialsEndpoint, relativeURI string) string { return credentialsEndpoint + relativeURI } + +// Based on old credentials.NewChainCredentials in v1 +func newChainCredentials(providers ...aws.CredentialsProvider) aws.CredentialsProvider { + return aws.NewCredentialsCache( + aws.CredentialsProviderFunc(func(ctx context.Context) (aws.Credentials, error) { + var errs []error + + for _, p := range providers { + creds, err := p.Retrieve(ctx) + if err == nil { + return creds, nil + } + + errs = append(errs, err) + } + + return aws.Credentials{}, fmt.Errorf("no valid providers in chain: %s", errs) + }), + ) +} diff --git a/pkg/storage/s3/s3_test.go b/pkg/storage/s3/s3_test.go index 08c7edd25..6d21f9523 100644 --- a/pkg/storage/s3/s3_test.go +++ b/pkg/storage/s3/s3_test.go @@ -2,12 +2,15 @@ package s3 import ( "context" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" + "github.com/gomods/athens/pkg/errors" "os" "testing" + "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/storage/compliance" ) @@ -26,7 +29,7 @@ func (s *Storage) clear() error { ctx, cancel := context.WithTimeout(context.Background(), s.timeout) defer cancel() - objects, err := s.s3API.ListObjectsWithContext(ctx, &s3.ListObjectsInput{Bucket: aws.String(s.bucket)}) + objects, err := s.s3API.ListObjects(ctx, &s3.ListObjectsInput{Bucket: aws.String(s.bucket)}) if err != nil { return err } @@ -37,7 +40,7 @@ func (s *Storage) clear() error { Key: o.Key, } - _, err := s.s3API.DeleteObjectWithContext(ctx, delParams) + _, err := s.s3API.DeleteObject(ctx, delParams) if err != nil { return err } @@ -49,23 +52,31 @@ func (s *Storage) createBucket() error { ctx, cancel := context.WithTimeout(context.Background(), s.timeout) defer cancel() - if _, err := s.s3API.CreateBucketWithContext(ctx, &s3.CreateBucketInput{Bucket: aws.String(s.bucket)}); err != nil { - aerr, ok := err.(awserr.Error) - if !ok { - return err + if _, err := s.s3API.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(s.bucket)}); err != nil { + var aerr smithy.APIError + + // aerr, ok := err.(awserr.Error) + // if !ok { + // return err + // } + + if errors.AsErr(err, &aerr) { + switch aerr.(type) { + case *types.BucketAlreadyOwnedByYou: + return nil + case *types.BucketAlreadyExists: + return nil + default: + return aerr + } } - switch aerr.Code() { - case s3.ErrCodeBucketAlreadyOwnedByYou: - return nil - case s3.ErrCodeBucketAlreadyExists: - return nil - default: - return aerr - } + return err } - return s.s3API.WaitUntilBucketExistsWithContext(ctx, &s3.HeadBucketInput{Bucket: aws.String(s.bucket)}) + waiter := s3.NewBucketExistsWaiter(s.s3API) + + return waiter.Wait(ctx, &s3.HeadBucketInput{Bucket: aws.String(s.bucket)}, 10*time.Minute) } func getStorage(t testing.TB) *Storage { @@ -75,9 +86,10 @@ func getStorage(t testing.TB) *Storage { } options := func(conf *aws.Config) { - conf.Endpoint = aws.String(url) - conf.DisableSSL = aws.Bool(true) + conf.BaseEndpoint = aws.String(url) + // conf.DisableSSL = aws.Bool(true) } + backend, err := New( &config.S3Config{ Key: "minio", diff --git a/pkg/storage/s3/saver.go b/pkg/storage/s3/saver.go index 990c15ca9..6bf0df678 100644 --- a/pkg/storage/s3/saver.go +++ b/pkg/storage/s3/saver.go @@ -3,10 +3,10 @@ package s3 import ( "bytes" "context" + "github.com/aws/aws-sdk-go-v2/service/s3" "io" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/observ" moduploader "github.com/gomods/athens/pkg/storage/module" @@ -31,14 +31,15 @@ func (s *Storage) upload(ctx context.Context, path, contentType string, stream i const op errors.Op = "s3.upload" ctx, span := observ.StartSpan(ctx, op.String()) defer span.End() - upParams := &s3manager.UploadInput{ + + upParams := &s3.PutObjectInput{ Bucket: aws.String(s.bucket), Key: aws.String(path), Body: stream, ContentType: aws.String(contentType), } - if _, err := s.uploader.UploadWithContext(ctx, upParams); err != nil { + if _, err := s.uploader.Upload(ctx, upParams); err != nil { return errors.E(op, err) } From 6d1ab09bfce82827b9f6be9f040fc4af2e397fcd Mon Sep 17 00:00:00 2001 From: root Date: Fri, 19 Apr 2024 09:40:30 +0700 Subject: [PATCH 02/14] Upgrade to AWS SDK v2 (#1938) --- pkg/storage/s3/s3.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pkg/storage/s3/s3.go b/pkg/storage/s3/s3.go index 66b1c94bf..241ffa461 100644 --- a/pkg/storage/s3/s3.go +++ b/pkg/storage/s3/s3.go @@ -24,11 +24,6 @@ import ( // - AWS_FORCE_PATH_STYLE - [optional] // For information how to get your keyId and access key turn to official aws docs: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/setting-up.html. -type s3API interface { - manager.DeleteObjectsAPIClient - manager.DownloadAPIClient - manager.ListObjectsV2APIClient -} type Storage struct { bucket string uploader *manager.Uploader From 4595adfdffbc14addb6f6295bc660ebe67a26567 Mon Sep 17 00:00:00 2001 From: k124k3n Date: Fri, 19 Apr 2024 12:14:34 +0700 Subject: [PATCH 03/14] Upgrade to AWS SDK v2 (#1938) --- cmd/proxy/actions/storage.go | 2 +- go.mod | 19 +++++++++++++++++ go.sum | 41 +++++++++++++++++++++++++++++++++--- pkg/storage/s3/checker.go | 6 +++--- pkg/storage/s3/s3_test.go | 6 ------ 5 files changed, 61 insertions(+), 13 deletions(-) diff --git a/cmd/proxy/actions/storage.go b/cmd/proxy/actions/storage.go index 3d9faa3eb..dd4e4164a 100644 --- a/cmd/proxy/actions/storage.go +++ b/cmd/proxy/actions/storage.go @@ -6,7 +6,7 @@ import ( "net/http" "time" - "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/storage" diff --git a/go.mod b/go.mod index 36c0c7de6..6f88317e8 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,12 @@ require ( github.com/BurntSushi/toml v1.0.0 github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20180917103902-e6c7f767dc57 github.com/aws/aws-sdk-go v1.44.220 + github.com/aws/aws-sdk-go-v2 v1.26.1 + github.com/aws/aws-sdk-go-v2/config v1.27.11 + github.com/aws/aws-sdk-go-v2/credentials v1.17.11 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15 + github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 + github.com/aws/smithy-go v1.20.2 github.com/bsm/redislock v0.7.2 github.com/fatih/color v1.13.0 github.com/go-playground/validator/v10 v10.19.0 @@ -55,6 +61,19 @@ require ( github.com/agext/levenshtein v1.2.1 // indirect github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f // indirect github.com/apparentlymart/go-textseg v1.0.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.1.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect diff --git a/go.sum b/go.sum index cea8aeeac..843969e0a 100644 --- a/go.sum +++ b/go.sum @@ -99,6 +99,44 @@ github.com/apparentlymart/go-textseg v1.0.0 h1:rRmlIsPEEhUTIKQb7T++Nz/A5Q6C9IuX2 github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk= github.com/aws/aws-sdk-go v1.44.220 h1:yAj99qAt0Htjle9Up3DglgHfOP77lmFPrElA4jKnrBo= github.com/aws/aws-sdk-go v1.44.220/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA= +github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg= +github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA= +github.com/aws/aws-sdk-go-v2/config v1.27.11/go.mod h1:SMsV78RIOYdve1vf36z8LmnszlRWkwMQtomCAI0/mIE= +github.com/aws/aws-sdk-go-v2/credentials v1.17.11 h1:YuIB1dJNf1Re822rriUOTxopaHHvIq0l/pX3fwO+Tzs= +github.com/aws/aws-sdk-go-v2/credentials v1.17.11/go.mod h1:AQtFPsDH9bI2O+71anW6EKL+NcD7LG3dpKGMV4SShgo= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 h1:FVJ0r5XTHSmIHJV6KuDmdYhEpvlHpiSd38RQWhut5J4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1/go.mod h1:zusuAeqezXzAB24LGuzuekqMAEgWkVYukBec3kr3jUg= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15 h1:7Zwtt/lP3KNRkeZre7soMELMGNoBrutx8nobg1jKWmo= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.16.15/go.mod h1:436h2adoHb57yd+8W+gYPrrA9U/R/SuAuOO42Ushzhw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 h1:aw39xVGeRWlWx9EzGVnhOR4yOjQDHPQ6o6NmBlscyQg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5/go.mod h1:FSaRudD0dXiMPK2UjknVwwTYyZMRsHv3TtkabsZih5I= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 h1:PG1F3OD1szkuQPzDw3CIQsRIrtTlUC3lP84taWzHlq0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5/go.mod h1:jU1li6RFryMz+so64PpKtudI+QzbKoIEivqdf6LNpOc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 h1:81KE7vaZzrl7yHBYHVEzYB8sypz11NMOZ40YlWvPxsU= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5/go.mod h1:LIt2rg7Mcgn09Ygbdh/RdIm0rQ+3BNkbP1gyVMFtRK0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 h1:ZMeFZ5yk+Ek+jNr1+uwCd2tG89t6oTS5yVWpa6yy2es= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7/go.mod h1:mxV05U+4JiHqIpGqqYXOHLPKUC6bDXC44bsUhNjOEwY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/gxJBcSWDMZlgyFUM962F51A5CRhDLbxLdmo= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 h1:f9RyWNtS8oH7cZlbn+/JNPpjUk5+5fLd5lM9M0i49Ys= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5/go.mod h1:h5CoMZV2VF297/VLhRhO1WF+XYWOzXo+4HsObA4HjBQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 h1:6cnno47Me9bRykw9AEv9zkXE+5or7jz8TsskTTccbgc= +github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1/go.mod h1:qmdkIIAC+GCLASF7R2whgNrJADz0QZPX+Seiw/i4S3o= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 h1:vN8hEbpRnL7+Hopy9dzmRle1xmDc7o8tmY0klsr175w= +github.com/aws/aws-sdk-go-v2/service/sso v1.20.5/go.mod h1:qGzynb/msuZIE8I75DVRCUXw3o3ZyBmUvMwQ2t/BrGM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 h1:Jux+gDDyi1Lruk+KHF91tK2KCuY61kzoCpvtvJJBtOE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4/go.mod h1:mUYPBhaF2lGiukDEjJX2BLRRKTmoUSitGDUgM4tRxak= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 h1:cwIxeBttqPN3qkaAjcEcsh8NYr8n2HZPkcKgPAi1phU= +github.com/aws/aws-sdk-go-v2/service/sts v1.28.6/go.mod h1:FZf1/nKNEkHdGGJP/cI2MoIMquumuRK6ol3QQJNDxmw= +github.com/aws/smithy-go v1.20.2 h1:tbp628ireGtzcHDDmLT/6ADHidqnwgF57XOXZe6tp4Q= +github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -441,8 +479,6 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= @@ -696,7 +732,6 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/pkg/storage/s3/checker.go b/pkg/storage/s3/checker.go index b447c6dcd..af5e36092 100644 --- a/pkg/storage/s3/checker.go +++ b/pkg/storage/s3/checker.go @@ -2,7 +2,7 @@ package s3 import ( "context" - "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/smithy-go" "sync" "github.com/aws/aws-sdk-go-v2/aws" @@ -44,8 +44,8 @@ func (s *Storage) Exists(ctx context.Context, module, version string) (bool, err if err == nil { continue } - var aerr awserr.Error - if errors.AsErr(err, &aerr) && aerr.Code() == "NotFound" { + var aerr smithy.APIError + if errors.AsErr(err, &aerr) && aerr.ErrorCode() == "NotFound" { err = nil exists = false } diff --git a/pkg/storage/s3/s3_test.go b/pkg/storage/s3/s3_test.go index 6d21f9523..4aaf8a067 100644 --- a/pkg/storage/s3/s3_test.go +++ b/pkg/storage/s3/s3_test.go @@ -55,11 +55,6 @@ func (s *Storage) createBucket() error { if _, err := s.s3API.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(s.bucket)}); err != nil { var aerr smithy.APIError - // aerr, ok := err.(awserr.Error) - // if !ok { - // return err - // } - if errors.AsErr(err, &aerr) { switch aerr.(type) { case *types.BucketAlreadyOwnedByYou: @@ -87,7 +82,6 @@ func getStorage(t testing.TB) *Storage { options := func(conf *aws.Config) { conf.BaseEndpoint = aws.String(url) - // conf.DisableSSL = aws.Bool(true) } backend, err := New( From f0582e9e68c8cbc43d6890bccc85346d6725da6f Mon Sep 17 00:00:00 2001 From: k124k3n Date: Sat, 20 Apr 2024 19:52:07 +0700 Subject: [PATCH 04/14] test(minio): add TrimHTTP to minio_test --- pkg/storage/minio/minio_test.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/pkg/storage/minio/minio_test.go b/pkg/storage/minio/minio_test.go index 85c87473f..35c73eb89 100644 --- a/pkg/storage/minio/minio_test.go +++ b/pkg/storage/minio/minio_test.go @@ -2,6 +2,7 @@ package minio import ( "os" + "strings" "testing" "github.com/gomods/athens/pkg/config" @@ -15,7 +16,7 @@ func TestBackend(t *testing.T) { // TestNewStorageExists tests the logic around MakeBucket and BucketExists func TestNewStorageExists(t *testing.T) { - url := os.Getenv("ATHENS_MINIO_ENDPOINT") + url := TrimHTTP(os.Getenv("ATHENS_MINIO_ENDPOINT")) if url == "" { t.SkipNow() } @@ -51,7 +52,7 @@ func TestNewStorageExists(t *testing.T) { // To ensure both paths are tested, there is a strict path error using the // "_" and a non strict error using less than 3 characters func TestNewStorageError(t *testing.T) { - url := os.Getenv("ATHENS_MINIO_ENDPOINT") + url := TrimHTTP(os.Getenv("ATHENS_MINIO_ENDPOINT")) if url == "" { t.SkipNow() } @@ -92,7 +93,7 @@ func (s *storageImpl) clear() error { } func getStorage(t testing.TB) *storageImpl { - url := os.Getenv("ATHENS_MINIO_ENDPOINT") + url := TrimHTTP(os.Getenv("ATHENS_MINIO_ENDPOINT")) if url == "" { t.SkipNow() } @@ -109,3 +110,10 @@ func getStorage(t testing.TB) *storageImpl { return backend.(*storageImpl) } + +// TrimHTTP trims "http://" or "https://" prefix from input string +func TrimHTTP(s string) string { + s = strings.TrimPrefix(s, "http://") + s = strings.TrimPrefix(s, "https://") + return s +} From e6a07d2ccfb495dcc845e2fc61ba2ea49713bf09 Mon Sep 17 00:00:00 2001 From: k124k3n Date: Sat, 20 Apr 2024 19:53:08 +0700 Subject: [PATCH 05/14] test(athens): add http:// to ATHENS_MINIO_ENDPOINT --- scripts/test_unit.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test_unit.sh b/scripts/test_unit.sh index 5451bd06c..57d982e2c 100755 --- a/scripts/test_unit.sh +++ b/scripts/test_unit.sh @@ -7,7 +7,7 @@ if [ -z ${GO_ENV} ]; then fi if [ -z ${ATHENS_MINIO_ENDPOINT} ]; then - export ATHENS_MINIO_ENDPOINT="127.0.0.1:9001" + export ATHENS_MINIO_ENDPOINT="http://127.0.0.1:9001" fi if [ -z ${ATHENS_MONGO_STORAGE_URL} ]; then From 801425061fd25b357d244018de5719b2765a339b Mon Sep 17 00:00:00 2001 From: k124k3n Date: Sat, 20 Apr 2024 20:02:14 +0700 Subject: [PATCH 06/14] chore(deps): tidy up go.mod --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 6f88317e8..8bbe4b832 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,6 @@ require ( github.com/Azure/azure-storage-blob-go v0.10.0 github.com/BurntSushi/toml v1.0.0 github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20180917103902-e6c7f767dc57 - github.com/aws/aws-sdk-go v1.44.220 github.com/aws/aws-sdk-go-v2 v1.26.1 github.com/aws/aws-sdk-go-v2/config v1.27.11 github.com/aws/aws-sdk-go-v2/credentials v1.17.11 @@ -61,6 +60,7 @@ require ( github.com/agext/levenshtein v1.2.1 // indirect github.com/ajg/form v0.0.0-20160822230020-523a5da1a92f // indirect github.com/apparentlymart/go-textseg v1.0.0 // indirect + github.com/aws/aws-sdk-go v1.44.220 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect From ebf1f4d5a471d22f4a18a255329bcddb32758662 Mon Sep 17 00:00:00 2001 From: k124k3n Date: Sat, 20 Apr 2024 20:18:25 +0700 Subject: [PATCH 07/14] test(athens): add http:// into ATHENS_MINIO_ENDPOINT --- .github/workflows/ci.yml | 2 +- docker-compose.yml | 2 +- scripts/ps/test_unit.ps1 | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e649b249..b91d6fa26 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: build: env: ATHENS_MONGO_STORAGE_URL: mongodb://localhost:27017 - ATHENS_MINIO_ENDPOINT: localhost:9000 + ATHENS_MINIO_ENDPOINT: http://localhost:9000 REDIS_TEST_ENDPOINT: localhost:6379 REDIS_SENTINEL_TEST_ENDPOINT: localhost:26379 REDIS_SENTINEL_TEST_MASTER_NAME: redis-1 diff --git a/docker-compose.yml b/docker-compose.yml index 67ddf92ac..b86946afc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,7 +24,7 @@ services: command: ["./scripts/test_unit.sh"] environment: - GO_ENV=test - - ATHENS_MINIO_ENDPOINT=minio:9000 + - ATHENS_MINIO_ENDPOINT=http://minio:9000 - ATHENS_MONGO_STORAGE_URL=mongodb://mongo:27017 - TIMEOUT=20 # in case the mongo dependency takes longer to start up - ATHENS_STORAGE_TYPE=mongo diff --git a/scripts/ps/test_unit.ps1 b/scripts/ps/test_unit.ps1 index ea7ee96ef..8a09db33a 100755 --- a/scripts/ps/test_unit.ps1 +++ b/scripts/ps/test_unit.ps1 @@ -3,7 +3,7 @@ if (!(Test-Path env:GO_ENV)) {$env:GO_ENV = "test"} if (!(Test-Path env:ATHENS_MINIO_ENDPOINT)) { - $env:ATHENS_MINIO_ENDPOINT = "127.0.0.1:9001" + $env:ATHENS_MINIO_ENDPOINT = "http://127.0.0.1:9001" } if (!(Test-Path env:ATHENS_MONGO_STORAGE_URL)) { From 74954ba72d56362de0cfb92b544e1fa483630c13 Mon Sep 17 00:00:00 2001 From: k124k3n Date: Sat, 20 Apr 2024 20:19:23 +0700 Subject: [PATCH 08/14] feat(minio): add TrimHTTP in endpoint assignment --- pkg/storage/minio/minio.go | 10 +++++++++- pkg/storage/minio/minio_test.go | 8 -------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/storage/minio/minio.go b/pkg/storage/minio/minio.go index b105aea18..18cfe6f17 100644 --- a/pkg/storage/minio/minio.go +++ b/pkg/storage/minio/minio.go @@ -2,6 +2,7 @@ package minio import ( "fmt" + "strings" "time" "github.com/gomods/athens/pkg/config" @@ -24,7 +25,7 @@ func (s *storageImpl) versionLocation(module, version string) string { // that implements storage.Backend. func NewStorage(conf *config.MinioConfig, timeout time.Duration) (storage.Backend, error) { const op errors.Op = "minio.NewStorage" - endpoint := conf.Endpoint + endpoint := TrimHTTP(conf.Endpoint) accessKeyID := conf.Key secretAccessKey := conf.Secret bucketName := conf.Bucket @@ -53,3 +54,10 @@ func NewStorage(conf *config.MinioConfig, timeout time.Duration) (storage.Backen } return &storageImpl{minioClient, minioCore, bucketName}, nil } + +// TrimHTTP trims "http://" or "https://" prefix from input string +func TrimHTTP(s string) string { + s = strings.TrimPrefix(s, "http://") + s = strings.TrimPrefix(s, "https://") + return s +} diff --git a/pkg/storage/minio/minio_test.go b/pkg/storage/minio/minio_test.go index 35c73eb89..bfe880c19 100644 --- a/pkg/storage/minio/minio_test.go +++ b/pkg/storage/minio/minio_test.go @@ -2,7 +2,6 @@ package minio import ( "os" - "strings" "testing" "github.com/gomods/athens/pkg/config" @@ -110,10 +109,3 @@ func getStorage(t testing.TB) *storageImpl { return backend.(*storageImpl) } - -// TrimHTTP trims "http://" or "https://" prefix from input string -func TrimHTTP(s string) string { - s = strings.TrimPrefix(s, "http://") - s = strings.TrimPrefix(s, "https://") - return s -} From 5dacd5a9c9266ec3ef2c4778e566f318a4691263 Mon Sep 17 00:00:00 2001 From: k124k3n Date: Sat, 20 Apr 2024 20:41:09 +0700 Subject: [PATCH 09/14] chore(lint): lint code --- cmd/proxy/actions/app_proxy_test.go | 3 +-- cmd/proxy/actions/index_test.go | 1 - e2etests/all_test.go | 6 +++--- pkg/config/config_test.go | 7 +++---- pkg/download/list_merge_test.go | 6 ++++-- pkg/download/mode/mode_test.go | 3 ++- pkg/download/protocol_test.go | 6 ++---- pkg/middleware/middleware_test.go | 2 +- pkg/module/filter_test.go | 10 ++++------ pkg/storage/minio/minio.go | 2 +- pkg/storage/mongo/mongo_test.go | 8 +++++--- pkg/storage/s3/cataloger.go | 3 ++- pkg/storage/s3/checker.go | 3 ++- pkg/storage/s3/deleter.go | 1 + pkg/storage/s3/getter.go | 3 ++- pkg/storage/s3/lister.go | 3 ++- pkg/storage/s3/s3.go | 10 +++++----- pkg/storage/s3/s3_test.go | 7 ++++--- pkg/storage/s3/saver.go | 3 ++- 19 files changed, 46 insertions(+), 41 deletions(-) diff --git a/cmd/proxy/actions/app_proxy_test.go b/cmd/proxy/actions/app_proxy_test.go index 735aea4dd..37b253d56 100644 --- a/cmd/proxy/actions/app_proxy_test.go +++ b/cmd/proxy/actions/app_proxy_test.go @@ -48,7 +48,7 @@ func TestProxyRoutes(t *testing.T) { tmp, err := template.New("home").Parse(homepage) assert.NoError(t, err) - var templateData = make(map[string]string) + templateData := make(map[string]string) templateData["Host"] = req.Host @@ -109,5 +109,4 @@ func TestProxyRoutes(t *testing.T) { tc.test(t, req, w.Result()) }) } - } diff --git a/cmd/proxy/actions/index_test.go b/cmd/proxy/actions/index_test.go index fdb1bf0dc..6e1692afb 100644 --- a/cmd/proxy/actions/index_test.go +++ b/cmd/proxy/actions/index_test.go @@ -107,7 +107,6 @@ func TestIndexHandler(t *testing.T) { if w.Code != tc.code { t.Fatalf("expected response code to be %d but got %d", tc.code, w.Code) } - }) } } diff --git a/e2etests/all_test.go b/e2etests/all_test.go index 27fc9c677..37c88f9f3 100644 --- a/e2etests/all_test.go +++ b/e2etests/all_test.go @@ -62,9 +62,9 @@ func (m *E2eSuite) SetupSuite() { func (m *E2eSuite) TearDownSuite() { m.stopAthens() - chmodR(m.goPath, 0777) + chmodR(m.goPath, 0o777) os.RemoveAll(m.goPath) - chmodR(m.sampleRepoPath, 0777) + chmodR(m.sampleRepoPath, 0o777) os.RemoveAll(m.sampleRepoPath) } @@ -73,7 +73,7 @@ func TestE2E(t *testing.T) { } func (m *E2eSuite) SetupTest() { - chmodR(m.goPath, 0777) + chmodR(m.goPath, 0o777) err := cleanGoCache(m.getEnv()) if err != nil { m.Fail("Failed to clear go cache", err) diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index c2897b568..74230e988 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -16,7 +16,7 @@ import ( func testConfigFile(t *testing.T) (testConfigFile string) { testConfigFile = filepath.Join("..", "..", "config.dev.toml") - if err := os.Chmod(testConfigFile, 0700); err != nil { + if err := os.Chmod(testConfigFile, 0o700); err != nil { t.Fatalf("%s\n", err) } return testConfigFile @@ -305,7 +305,6 @@ func TestParseExampleConfig(t *testing.T) { } func getEnvMap(config *Config) map[string]string { - envVars := map[string]string{ "GO_ENV": config.GoEnv, "GO_BINARY_PATH": config.GoBinary, @@ -414,7 +413,7 @@ func Test_checkFilePerms(t *testing.T) { } incorrectPerms := []os.FileMode{0o777, 0o610, 0o660} - var incorrectFiles = make([]string, len(incorrectPerms)) + incorrectFiles := make([]string, len(incorrectPerms)) for i := range incorrectPerms { f, err := tempFile(incorrectPerms[i]) @@ -426,7 +425,7 @@ func Test_checkFilePerms(t *testing.T) { } correctPerms := []os.FileMode{0o600, 0o400, 0o644} - var correctFiles = make([]string, len(correctPerms)) + correctFiles := make([]string, len(correctPerms)) for i := range correctPerms { f, err := tempFile(correctPerms[i]) diff --git a/pkg/download/list_merge_test.go b/pkg/download/list_merge_test.go index 3ba65190e..63b2c5818 100644 --- a/pkg/download/list_merge_test.go +++ b/pkg/download/list_merge_test.go @@ -13,8 +13,10 @@ import ( "github.com/stretchr/testify/require" ) -const testOp athenserr.Op = "vcsLister.List" -const testModName = "happy tags" +const ( + testOp athenserr.Op = "vcsLister.List" + testModName = "happy tags" +) type listMergeTest struct { name string diff --git a/pkg/download/mode/mode_test.go b/pkg/download/mode/mode_test.go index abb0355c4..a02d38863 100644 --- a/pkg/download/mode/mode_test.go +++ b/pkg/download/mode/mode_test.go @@ -84,7 +84,8 @@ var testCases = []struct { { Pattern: "github.com/gomods/*", Mode: AsyncRedirect, - DownloadURL: "gomods.io"}, + DownloadURL: "gomods.io", + }, }, }, input: "github.com/gomods/athens", diff --git a/pkg/download/protocol_test.go b/pkg/download/protocol_test.go index d2f910684..1a8559842 100644 --- a/pkg/download/protocol_test.go +++ b/pkg/download/protocol_test.go @@ -28,9 +28,7 @@ import ( "golang.org/x/sync/errgroup" ) -var ( - testConfigPath = filepath.Join("..", "..", "config.dev.toml") -) +var testConfigPath = filepath.Join("..", "..", "config.dev.toml") func getDP(t *testing.T) Protocol { t.Helper() @@ -235,7 +233,7 @@ var latestTests = []latestTest{ path: "github.com/athens-artifacts/happy-path", info: &storage.RevInfo{ Version: "v0.0.3", - Time: time.Date(2018, 8, 3, 17, 16, 00, 0, time.UTC), + Time: time.Date(2018, 8, 3, 17, 16, 0o0, 0, time.UTC), }, }, } diff --git a/pkg/middleware/middleware_test.go b/pkg/middleware/middleware_test.go index 3928fdb98..46d87f6d2 100644 --- a/pkg/middleware/middleware_test.go +++ b/pkg/middleware/middleware_test.go @@ -23,7 +23,7 @@ const ( func testConfigFile(t *testing.T) (testConfigFile string) { testConfigFile = filepath.Join("..", "..", "config.dev.toml") - if err := os.Chmod(testConfigFile, 0700); err != nil { + if err := os.Chmod(testConfigFile, 0o700); err != nil { t.Fatalf("%s\n", err) } return testConfigFile diff --git a/pkg/module/filter_test.go b/pkg/module/filter_test.go index 1df3e9475..9a8e1d75e 100644 --- a/pkg/module/filter_test.go +++ b/pkg/module/filter_test.go @@ -10,7 +10,7 @@ import ( func testConfigFile(t *testing.T) (testConfigFile string) { testConfigFile = filepath.Join("..", "..", "config.dev.toml") - if err := os.Chmod(testConfigFile, 0700); err != nil { + if err := os.Chmod(testConfigFile, 0o700); err != nil { t.Fatalf("%s\n", err) } return testConfigFile @@ -40,7 +40,6 @@ func (t *FilterTests) Test_NewFilter() { mf, err = NewFilter(filter) r.Equal(filter, mf.filePath) r.NoError(err) - } func (t *FilterTests) Test_IgnoreSimple() { @@ -198,7 +197,6 @@ func (t *FilterTests) Test_versionFilterRobust() { r.Equal(Exclude, f.Rule("github.com/a/b", "a")) r.Equal(Exclude, f.Rule("github.com/c/d", "fg")) - } func (t *FilterTests) Test_initFromConfig() { @@ -207,20 +205,20 @@ func (t *FilterTests) Test_initFromConfig() { defer os.Remove(filterFile) goodInput := []byte("+ github.com/a/b\n\n# some comment\n- github.com/c/d\n\nD github.com/x") - os.WriteFile(filterFile, goodInput, 0644) + os.WriteFile(filterFile, goodInput, 0o644) f, err := initFromConfig(filterFile) r.NotNil(f) r.NoError(err) badInput := []byte("+ github.com/a/b\n\n# some comment\n\n- github.com/c/d\n\nD github.com/x\nsome_random_line") - os.WriteFile(filterFile, badInput, 0644) + os.WriteFile(filterFile, badInput, 0o644) f, err = initFromConfig(filterFile) r.Nil(f) r.Error(err) versionInput := []byte("+ github.com/a/b\n\n# some comment\n\n- github.com/c/d v1,v2.3.4,v3.2.*\n\nD github.com/x\n") - os.WriteFile(filterFile, versionInput, 0644) + os.WriteFile(filterFile, versionInput, 0o644) f, err = initFromConfig(filterFile) r.NotNil(f) r.NoError(err) diff --git a/pkg/storage/minio/minio.go b/pkg/storage/minio/minio.go index 18cfe6f17..5336728bf 100644 --- a/pkg/storage/minio/minio.go +++ b/pkg/storage/minio/minio.go @@ -55,7 +55,7 @@ func NewStorage(conf *config.MinioConfig, timeout time.Duration) (storage.Backen return &storageImpl{minioClient, minioCore, bucketName}, nil } -// TrimHTTP trims "http://" or "https://" prefix from input string +// TrimHTTP trims "http://" or "https://" prefix from input string. func TrimHTTP(s string) string { s = strings.TrimPrefix(s, "http://") s = strings.TrimPrefix(s, "https://") diff --git a/pkg/storage/mongo/mongo_test.go b/pkg/storage/mongo/mongo_test.go index 7f52f2522..3f72ab0a7 100644 --- a/pkg/storage/mongo/mongo_test.go +++ b/pkg/storage/mongo/mongo_test.go @@ -3,13 +3,14 @@ package mongo import ( "bytes" "context" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" "io" "os" "testing" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/storage" @@ -145,6 +146,7 @@ func TestQueryKindUnexpectedErrorCases(t *testing.T) { require.Equal(t, errors.KindUnexpected, errors.Kind(err)) } } + func TestNewStorageWithDefaultOverrides(t *testing.T) { url := os.Getenv("ATHENS_MONGO_STORAGE_URL") diff --git a/pkg/storage/s3/cataloger.go b/pkg/storage/s3/cataloger.go index f1e591042..fac0d429e 100644 --- a/pkg/storage/s3/cataloger.go +++ b/pkg/storage/s3/cataloger.go @@ -3,9 +3,10 @@ package s3 import ( "context" "fmt" - "github.com/aws/aws-sdk-go-v2/service/s3/types" "strings" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/gomods/athens/pkg/config" diff --git a/pkg/storage/s3/checker.go b/pkg/storage/s3/checker.go index af5e36092..59d3b11fe 100644 --- a/pkg/storage/s3/checker.go +++ b/pkg/storage/s3/checker.go @@ -2,9 +2,10 @@ package s3 import ( "context" - "github.com/aws/smithy-go" "sync" + "github.com/aws/smithy-go" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/gomods/athens/pkg/config" diff --git a/pkg/storage/s3/deleter.go b/pkg/storage/s3/deleter.go index 4651cefed..4e1201ed3 100644 --- a/pkg/storage/s3/deleter.go +++ b/pkg/storage/s3/deleter.go @@ -2,6 +2,7 @@ package s3 import ( "context" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/gomods/athens/pkg/errors" diff --git a/pkg/storage/s3/getter.go b/pkg/storage/s3/getter.go index ab01b7e98..1c14a1bcb 100644 --- a/pkg/storage/s3/getter.go +++ b/pkg/storage/s3/getter.go @@ -3,9 +3,10 @@ package s3 import ( "context" "fmt" - "github.com/aws/aws-sdk-go-v2/service/s3/types" "io" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/gomods/athens/pkg/config" diff --git a/pkg/storage/s3/lister.go b/pkg/storage/s3/lister.go index e593e88ea..15590e8ce 100644 --- a/pkg/storage/s3/lister.go +++ b/pkg/storage/s3/lister.go @@ -2,9 +2,10 @@ package s3 import ( "context" - "github.com/aws/aws-sdk-go-v2/service/s3/types" "strings" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/gomods/athens/pkg/errors" diff --git a/pkg/storage/s3/s3.go b/pkg/storage/s3/s3.go index 241ffa461..a431e6a60 100644 --- a/pkg/storage/s3/s3.go +++ b/pkg/storage/s3/s3.go @@ -3,6 +3,8 @@ package s3 import ( "context" "fmt" + "time" + "github.com/aws/aws-sdk-go-v2/aws" awscfg "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" @@ -11,7 +13,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/errors" - "time" ) // Storage implements (./pkg/storage).Backend and @@ -36,7 +37,6 @@ func New(s3Conf *config.S3Config, timeout time.Duration, options ...func(*aws.Co const op errors.Op = "s3.New" awsConfig, err := awscfg.LoadDefaultConfig(context.TODO(), awscfg.WithRegion(s3Conf.Region)) - if err != nil { return nil, errors.E(op, err) } @@ -50,13 +50,13 @@ func New(s3Conf *config.S3Config, timeout time.Duration, options ...func(*aws.Co } if !s3Conf.UseDefaultConfiguration { - //credProviders := defaults.CredProviders(awsConfig, defaults.Handlers()) + // credProviders := defaults.CredProviders(awsConfig, defaults.Handlers()) endpointCreds := []aws.CredentialsProvider{ endpointcreds.New(endpointFrom(s3Conf.CredentialsEndpoint, s3Conf.AwsContainerCredentialsRelativeURI)), credentials.NewStaticCredentialsProvider(s3Conf.Key, s3Conf.Secret, s3Conf.Token), } - //credProviders = append(endpointCreds, credProviders...) + // credProviders = append(endpointCreds, credProviders...) awsConfig.Credentials = newChainCredentials(endpointCreds...) } @@ -82,7 +82,7 @@ func endpointFrom(credentialsEndpoint, relativeURI string) string { return credentialsEndpoint + relativeURI } -// Based on old credentials.NewChainCredentials in v1 +// newChainCredentials is based on old credentials.NewChainCredentials in v1. func newChainCredentials(providers ...aws.CredentialsProvider) aws.CredentialsProvider { return aws.NewCredentialsCache( aws.CredentialsProviderFunc(func(ctx context.Context) (aws.Credentials, error) { diff --git a/pkg/storage/s3/s3_test.go b/pkg/storage/s3/s3_test.go index 4aaf8a067..571d30e57 100644 --- a/pkg/storage/s3/s3_test.go +++ b/pkg/storage/s3/s3_test.go @@ -2,13 +2,14 @@ package s3 import ( "context" - "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/aws/smithy-go" - "github.com/gomods/athens/pkg/errors" "os" "testing" "time" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" + "github.com/gomods/athens/pkg/errors" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/gomods/athens/pkg/config" diff --git a/pkg/storage/s3/saver.go b/pkg/storage/s3/saver.go index 6bf0df678..951c90ec6 100644 --- a/pkg/storage/s3/saver.go +++ b/pkg/storage/s3/saver.go @@ -3,9 +3,10 @@ package s3 import ( "bytes" "context" - "github.com/aws/aws-sdk-go-v2/service/s3" "io" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/observ" From 5bfc6f4b4fc157e86666e50d65eb1e33a0ca13e5 Mon Sep 17 00:00:00 2001 From: k124k3n Date: Sat, 20 Apr 2024 21:25:02 +0700 Subject: [PATCH 10/14] chore(lint): lint code --- go.sum | 4 ++++ pkg/config/azureblob.go | 6 +++--- pkg/config/config.go | 20 ++++++++++---------- pkg/config/disk.go | 2 +- pkg/config/external.go | 2 +- pkg/config/gcp.go | 2 +- pkg/config/minio.go | 8 ++++---- pkg/config/mongo.go | 6 +++--- pkg/config/mysql.go | 14 +++++++------- pkg/config/postgres.go | 12 ++++++------ pkg/config/s3.go | 4 ++-- pkg/storage/s3/cataloger.go | 3 +-- pkg/storage/s3/checker.go | 3 +-- pkg/storage/s3/getter.go | 3 +-- pkg/storage/s3/lister.go | 3 +-- pkg/storage/s3/saver.go | 3 +-- 16 files changed, 47 insertions(+), 48 deletions(-) diff --git a/go.sum b/go.sum index 4ce6080d5..843969e0a 100644 --- a/go.sum +++ b/go.sum @@ -687,6 +687,7 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -770,10 +771,12 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -783,6 +786,7 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/pkg/config/azureblob.go b/pkg/config/azureblob.go index bbff569f7..5935db957 100644 --- a/pkg/config/azureblob.go +++ b/pkg/config/azureblob.go @@ -2,7 +2,7 @@ package config // AzureBlobConfig specifies the properties required to use Azure as the storage backend. type AzureBlobConfig struct { - AccountName string `validate:"required" envconfig:"ATHENS_AZURE_ACCOUNT_NAME"` - AccountKey string `validate:"required" envconfig:"ATHENS_AZURE_ACCOUNT_KEY"` - ContainerName string `validate:"required" envconfig:"ATHENS_AZURE_CONTAINER_NAME"` + AccountName string `envconfig:"ATHENS_AZURE_ACCOUNT_NAME" validate:"required"` + AccountKey string `envconfig:"ATHENS_AZURE_ACCOUNT_KEY" validate:"required"` + ContainerName string `envconfig:"ATHENS_AZURE_CONTAINER_NAME" validate:"required"` } diff --git a/pkg/config/config.go b/pkg/config/config.go index 9f070e5b1..9caed7fee 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -21,22 +21,22 @@ const defaultConfigFile = "athens.toml" // Config provides configuration values for all components. type Config struct { TimeoutConf - GoEnv string `validate:"required" envconfig:"GO_ENV"` - GoBinary string `validate:"required" envconfig:"GO_BINARY_PATH"` + GoEnv string `envconfig:"GO_ENV" validate:"required"` + GoBinary string `envconfig:"GO_BINARY_PATH" validate:"required"` GoBinaryEnvVars EnvList `envconfig:"ATHENS_GO_BINARY_ENV_VARS"` - GoGetWorkers int `validate:"required" envconfig:"ATHENS_GOGET_WORKERS"` + GoGetWorkers int `envconfig:"ATHENS_GOGET_WORKERS" validate:"required"` GoGetDir string `envconfig:"ATHENS_GOGET_DIR"` - ProtocolWorkers int `validate:"required" envconfig:"ATHENS_PROTOCOL_WORKERS"` - LogLevel string `validate:"required" envconfig:"ATHENS_LOG_LEVEL"` - LogFormat string `validate:"oneof='' 'json' 'plain'" envconfig:"ATHENS_LOG_FORMAT"` - CloudRuntime string `validate:"required_without=LogFormat" envconfig:"ATHENS_CLOUD_RUNTIME"` + ProtocolWorkers int `envconfig:"ATHENS_PROTOCOL_WORKERS" validate:"required"` + LogLevel string `envconfig:"ATHENS_LOG_LEVEL" validate:"required"` + LogFormat string `envconfig:"ATHENS_LOG_FORMAT" validate:"oneof='' 'json' 'plain'"` + CloudRuntime string `envconfig:"ATHENS_CLOUD_RUNTIME" validate:"required_without=LogFormat"` EnablePprof bool `envconfig:"ATHENS_ENABLE_PPROF"` PprofPort string `envconfig:"ATHENS_PPROF_PORT"` FilterFile string `envconfig:"ATHENS_FILTER_FILE"` TraceExporterURL string `envconfig:"ATHENS_TRACE_EXPORTER_URL"` TraceExporter string `envconfig:"ATHENS_TRACE_EXPORTER"` StatsExporter string `envconfig:"ATHENS_STATS_EXPORTER"` - StorageType string `validate:"required" envconfig:"ATHENS_STORAGE_TYPE"` + StorageType string `envconfig:"ATHENS_STORAGE_TYPE" validate:"required"` GlobalEndpoint string `envconfig:"ATHENS_GLOBAL_ENDPOINT"` // This feature is not yet implemented Port string `envconfig:"ATHENS_PORT"` UnixSocket string `envconfig:"ATHENS_UNIX_SOCKET"` @@ -55,11 +55,11 @@ type Config struct { NoSumPatterns []string `envconfig:"ATHENS_GONOSUM_PATTERNS"` DownloadMode mode.Mode `envconfig:"ATHENS_DOWNLOAD_MODE"` DownloadURL string `envconfig:"ATHENS_DOWNLOAD_URL"` - NetworkMode string `validate:"oneof=strict offline fallback" envconfig:"ATHENS_NETWORK_MODE"` + NetworkMode string `envconfig:"ATHENS_NETWORK_MODE" validate:"oneof=strict offline fallback"` SingleFlightType string `envconfig:"ATHENS_SINGLE_FLIGHT_TYPE"` RobotsFile string `envconfig:"ATHENS_ROBOTS_FILE"` IndexType string `envconfig:"ATHENS_INDEX_TYPE"` - ShutdownTimeout int `validate:"min=0" envconfig:"ATHENS_SHUTDOWN_TIMEOUT"` + ShutdownTimeout int `envconfig:"ATHENS_SHUTDOWN_TIMEOUT" validate:"min=0"` SingleFlight *SingleFlight Storage *Storage Index *Index diff --git a/pkg/config/disk.go b/pkg/config/disk.go index 2db42f222..cb867d654 100644 --- a/pkg/config/disk.go +++ b/pkg/config/disk.go @@ -2,5 +2,5 @@ package config // DiskConfig specifies the properties required to use Disk as the storage backend. type DiskConfig struct { - RootPath string `validate:"required" envconfig:"ATHENS_DISK_STORAGE_ROOT"` + RootPath string `envconfig:"ATHENS_DISK_STORAGE_ROOT" validate:"required"` } diff --git a/pkg/config/external.go b/pkg/config/external.go index ff5f563fa..574a4ede2 100644 --- a/pkg/config/external.go +++ b/pkg/config/external.go @@ -2,5 +2,5 @@ package config // External specifies configuration for an external http storage. type External struct { - URL string `validate:"required" envconfig:"ATHENS_EXTERNAL_STORAGE_URL"` + URL string `envconfig:"ATHENS_EXTERNAL_STORAGE_URL" validate:"required"` } diff --git a/pkg/config/gcp.go b/pkg/config/gcp.go index 26f7343af..4ae634d95 100644 --- a/pkg/config/gcp.go +++ b/pkg/config/gcp.go @@ -3,6 +3,6 @@ package config // GCPConfig specifies the properties required to use GCP as the storage backend. type GCPConfig struct { ProjectID string `envconfig:"GOOGLE_CLOUD_PROJECT"` - Bucket string `validate:"required" envconfig:"ATHENS_STORAGE_GCP_BUCKET"` + Bucket string `envconfig:"ATHENS_STORAGE_GCP_BUCKET" validate:"required"` JSONKey string `envconfig:"ATHENS_STORAGE_GCP_JSON_KEY"` } diff --git a/pkg/config/minio.go b/pkg/config/minio.go index ee6243820..4147a577b 100644 --- a/pkg/config/minio.go +++ b/pkg/config/minio.go @@ -3,10 +3,10 @@ package config // MinioConfig specifies the properties required to use Minio or DigitalOcean Spaces // as the storage backend. type MinioConfig struct { - Endpoint string `validate:"required" envconfig:"ATHENS_MINIO_ENDPOINT"` - Key string `validate:"required" envconfig:"ATHENS_MINIO_ACCESS_KEY_ID"` - Secret string `validate:"required" envconfig:"ATHENS_MINIO_SECRET_ACCESS_KEY"` - Bucket string `validate:"required" envconfig:"ATHENS_MINIO_BUCKET_NAME"` + Endpoint string `envconfig:"ATHENS_MINIO_ENDPOINT" validate:"required"` + Key string `envconfig:"ATHENS_MINIO_ACCESS_KEY_ID" validate:"required"` + Secret string `envconfig:"ATHENS_MINIO_SECRET_ACCESS_KEY" validate:"required"` + Bucket string `envconfig:"ATHENS_MINIO_BUCKET_NAME" validate:"required"` Region string `envconfig:"ATHENS_MINIO_REGION"` EnableSSL bool `envconfig:"ATHENS_MINIO_USE_SSL"` } diff --git a/pkg/config/mongo.go b/pkg/config/mongo.go index 91f6f6549..527773344 100644 --- a/pkg/config/mongo.go +++ b/pkg/config/mongo.go @@ -2,9 +2,9 @@ package config // MongoConfig specifies the properties required to use MongoDB as the storage backend. type MongoConfig struct { - URL string `validate:"required" envconfig:"ATHENS_MONGO_STORAGE_URL"` - DefaultDBName string `envconfig:"ATHENS_MONGO_DEFAULT_DATABASE" default:"athens"` - DefaultCollectionName string `envconfig:"ATHENS_MONGO_DEFAULT_COLLECTION" default:"modules"` + URL string `envconfig:"ATHENS_MONGO_STORAGE_URL" validate:"required"` + DefaultDBName string `default:"athens" envconfig:"ATHENS_MONGO_DEFAULT_DATABASE"` + DefaultCollectionName string `default:"modules" envconfig:"ATHENS_MONGO_DEFAULT_COLLECTION"` CertPath string `envconfig:"ATHENS_MONGO_CERT_PATH"` InsecureConn bool `envconfig:"ATHENS_MONGO_INSECURE"` } diff --git a/pkg/config/mysql.go b/pkg/config/mysql.go index 46e5c959f..83b86be98 100644 --- a/pkg/config/mysql.go +++ b/pkg/config/mysql.go @@ -2,11 +2,11 @@ package config // MySQL config. type MySQL struct { - Protocol string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_PROTOCOL"` - Host string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_HOST"` - Port int `validate:"" envconfig:"ATHENS_INDEX_MYSQL_PORT"` - User string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_USER"` - Password string `validate:"" envconfig:"ATHENS_INDEX_MYSQL_PASSWORD"` - Database string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_DATABASE"` - Params map[string]string `validate:"required" envconfig:"ATHENS_INDEX_MYSQL_PARAMS"` + Protocol string `envconfig:"ATHENS_INDEX_MYSQL_PROTOCOL" validate:"required"` + Host string `envconfig:"ATHENS_INDEX_MYSQL_HOST" validate:"required"` + Port int `envconfig:"ATHENS_INDEX_MYSQL_PORT" validate:""` + User string `envconfig:"ATHENS_INDEX_MYSQL_USER" validate:"required"` + Password string `envconfig:"ATHENS_INDEX_MYSQL_PASSWORD" validate:""` + Database string `envconfig:"ATHENS_INDEX_MYSQL_DATABASE" validate:"required"` + Params map[string]string `envconfig:"ATHENS_INDEX_MYSQL_PARAMS" validate:"required"` } diff --git a/pkg/config/postgres.go b/pkg/config/postgres.go index ee2827f15..aa56f4176 100644 --- a/pkg/config/postgres.go +++ b/pkg/config/postgres.go @@ -2,10 +2,10 @@ package config // Postgres config. type Postgres struct { - Host string `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_HOST"` - Port int `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_PORT"` - User string `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_USER"` - Password string `validate:"" envconfig:"ATHENS_INDEX_POSTGRES_PASSWORD"` - Database string `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_DATABASE"` - Params map[string]string `validate:"required" envconfig:"ATHENS_INDEX_POSTGRES_PARAMS"` + Host string `envconfig:"ATHENS_INDEX_POSTGRES_HOST" validate:"required"` + Port int `envconfig:"ATHENS_INDEX_POSTGRES_PORT" validate:"required"` + User string `envconfig:"ATHENS_INDEX_POSTGRES_USER" validate:"required"` + Password string `envconfig:"ATHENS_INDEX_POSTGRES_PASSWORD" validate:""` + Database string `envconfig:"ATHENS_INDEX_POSTGRES_DATABASE" validate:"required"` + Params map[string]string `envconfig:"ATHENS_INDEX_POSTGRES_PARAMS" validate:"required"` } diff --git a/pkg/config/s3.go b/pkg/config/s3.go index f72faefd1..63d3fa1cc 100644 --- a/pkg/config/s3.go +++ b/pkg/config/s3.go @@ -2,11 +2,11 @@ package config // S3Config specifies the properties required to use S3 as the storage backend. type S3Config struct { - Region string `validate:"required" envconfig:"AWS_REGION"` + Region string `envconfig:"AWS_REGION" validate:"required"` Key string `envconfig:"AWS_ACCESS_KEY_ID"` Secret string `envconfig:"AWS_SECRET_ACCESS_KEY"` Token string `envconfig:"AWS_SESSION_TOKEN"` - Bucket string `validate:"required" envconfig:"ATHENS_S3_BUCKET_NAME"` + Bucket string `envconfig:"ATHENS_S3_BUCKET_NAME" validate:"required"` UseDefaultConfiguration bool `envconfig:"AWS_USE_DEFAULT_CONFIGURATION"` ForcePathStyle bool `envconfig:"AWS_FORCE_PATH_STYLE"` CredentialsEndpoint string `envconfig:"AWS_CREDENTIALS_ENDPOINT"` diff --git a/pkg/storage/s3/cataloger.go b/pkg/storage/s3/cataloger.go index fac0d429e..83706e180 100644 --- a/pkg/storage/s3/cataloger.go +++ b/pkg/storage/s3/cataloger.go @@ -5,10 +5,9 @@ import ( "fmt" "strings" - "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/observ" diff --git a/pkg/storage/s3/checker.go b/pkg/storage/s3/checker.go index 59d3b11fe..faa0f2f7b 100644 --- a/pkg/storage/s3/checker.go +++ b/pkg/storage/s3/checker.go @@ -4,10 +4,9 @@ import ( "context" "sync" - "github.com/aws/smithy-go" - "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" "github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/observ" diff --git a/pkg/storage/s3/getter.go b/pkg/storage/s3/getter.go index 1c14a1bcb..cedd1393d 100644 --- a/pkg/storage/s3/getter.go +++ b/pkg/storage/s3/getter.go @@ -5,10 +5,9 @@ import ( "fmt" "io" - "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/gomods/athens/pkg/config" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/observ" diff --git a/pkg/storage/s3/lister.go b/pkg/storage/s3/lister.go index 15590e8ce..43c29c4ce 100644 --- a/pkg/storage/s3/lister.go +++ b/pkg/storage/s3/lister.go @@ -4,10 +4,9 @@ import ( "context" "strings" - "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/observ" ) diff --git a/pkg/storage/s3/saver.go b/pkg/storage/s3/saver.go index 951c90ec6..cbf6b5d8c 100644 --- a/pkg/storage/s3/saver.go +++ b/pkg/storage/s3/saver.go @@ -5,9 +5,8 @@ import ( "context" "io" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/gomods/athens/pkg/errors" "github.com/gomods/athens/pkg/observ" moduploader "github.com/gomods/athens/pkg/storage/module" From b1bfd6517b94ea2da7b5ac109d207bc0bf97fcb1 Mon Sep 17 00:00:00 2001 From: k124k3n Date: Sat, 20 Apr 2024 21:27:42 +0700 Subject: [PATCH 11/14] chore(lint): lint code --- pkg/storage/s3/s3.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/storage/s3/s3.go b/pkg/storage/s3/s3.go index a431e6a60..8d648b120 100644 --- a/pkg/storage/s3/s3.go +++ b/pkg/storage/s3/s3.go @@ -24,7 +24,6 @@ import ( // - AWS_SESSION_TOKEN - [optional] // - AWS_FORCE_PATH_STYLE - [optional] // For information how to get your keyId and access key turn to official aws docs: https://docs.aws.amazon.com/sdk-for-go/v1/developer-guide/setting-up.html. - type Storage struct { bucket string uploader *manager.Uploader From 24a5601ca1579b63f434bc82900bf71ace9b17fc Mon Sep 17 00:00:00 2001 From: k124k3n Date: Sat, 20 Apr 2024 21:35:02 +0700 Subject: [PATCH 12/14] chore(lint): lint code --- pkg/config/config_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 74230e988..6c741a04a 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -108,6 +108,7 @@ func TestEnvOverrides(t *testing.T) { if err != nil { t.Fatalf("Env override failed: %v", err) } + compareConfigs(conf, expConf, t, Storage{}, SingleFlight{}) } From 63a0dbc2e711373e7886851520b4ffa3358d8ed1 Mon Sep 17 00:00:00 2001 From: Vanes Angelo <63584188+k124k3n@users.noreply.github.com> Date: Thu, 25 Apr 2024 14:23:09 +0700 Subject: [PATCH 13/14] docs: add docs in TrimHTTP --- pkg/storage/minio/minio.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/storage/minio/minio.go b/pkg/storage/minio/minio.go index 5336728bf..4d0c1ddff 100644 --- a/pkg/storage/minio/minio.go +++ b/pkg/storage/minio/minio.go @@ -56,6 +56,8 @@ func NewStorage(conf *config.MinioConfig, timeout time.Duration) (storage.Backen } // TrimHTTP trims "http://" or "https://" prefix from input string. +// Minio doesn't need to specify protocol in the URL so it should be trimmed.  +// Related issue: https://github.com/gomods/athens/issues/1938#issuecomment-2067590653 func TrimHTTP(s string) string { s = strings.TrimPrefix(s, "http://") s = strings.TrimPrefix(s, "https://") From 33256193270792daadd7c8c6745b6f38afb53c60 Mon Sep 17 00:00:00 2001 From: k124k3n Date: Thu, 25 Apr 2024 16:25:07 +0700 Subject: [PATCH 14/14] chore(lint): lint code --- pkg/storage/minio/minio.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/storage/minio/minio.go b/pkg/storage/minio/minio.go index 4d0c1ddff..8dbede3e9 100644 --- a/pkg/storage/minio/minio.go +++ b/pkg/storage/minio/minio.go @@ -56,7 +56,7 @@ func NewStorage(conf *config.MinioConfig, timeout time.Duration) (storage.Backen } // TrimHTTP trims "http://" or "https://" prefix from input string. -// Minio doesn't need to specify protocol in the URL so it should be trimmed.  +// Minio doesn't need to specify protocol in the URL so it should be trimmed. // Related issue: https://github.com/gomods/athens/issues/1938#issuecomment-2067590653 func TrimHTTP(s string) string { s = strings.TrimPrefix(s, "http://")