Use S3 client to call terrablob

Summary:
* change configuration to use S3 as the storage client
* configure parameters to call  Terrablob with S3 API according to https://engwiki.uberinternal.com/display/TERRABLOB/S3+HTTP+API
* Patch the S3 client to put all modules under a path prefix that we use in Terrablob
* Patch a pending pull request gomods#1720

Test Plan: * deployed to staging, change GOPROXY to point to staging, and run `go clean -modcache && go mod tidy`

Reviewers: tanx, sergeyb

Reviewed By: tanx, sergeyb

JIRA Issues: GM-673

Differential Revision: https://code.uberinternal.com/D6235753

pkg/config/module.go

@@ -6,10 +6,12 @@ import (
 	"strings"
 )
 
+var _terrablobPathPrefix = "/prod/build-platform/goproxy/v2/"
+
 // PackageVersionedName return package full name used in storage.
 // E.g athens/@v/v1.0.mod
 func PackageVersionedName(module, version, ext string) string {
-	return fmt.Sprintf("%s/@v/%s.%s", module, version, ext)
+	return fmt.Sprintf("%s%s/@v/%s.%s", _terrablobPathPrefix, module, version, ext)
 }
 
 // FmtModVer is a helper function that can take
@@ -22,6 +24,7 @@ func FmtModVer(mod, ver string) string {
 // storage path
 // E.g athens/@v/v1.0.info -> athens and v.1.0
 func ModuleVersionFromPath(path string) (string, string) {
+	path = strings.TrimPrefix(path, _terrablobPathPrefix)
 	segments := strings.Split(path, "/@v/")
 	if len(segments) != 2 {
 		return "", ""

pkg/storage/s3/checker.go

@@ -2,6 +2,7 @@ package s3
 
 import (
 	"context"
+	"path"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/s3"
@@ -23,7 +24,8 @@ func (s *Storage) Exists(ctx context.Context, module, version string) (bool, err
 
 	lsParams := &s3.ListObjectsInput{
 		Bucket: aws.String(s.bucket),
-		Prefix: aws.String(config.PackageVersionedName(module, version, "")),
+		// https://stack.uberinternal.com/questions/59998
+		Prefix: aws.String(path.Dir(config.PackageVersionedName(module, version, ""))),
 	}
 
 	loo, err := s.s3API.ListObjectsWithContext(ctx, lsParams)

uber.config.toml

@@ -116,7 +116,7 @@ Timeout = 300
 # Possible values are memory, disk, mongo, gcp, minio, s3, azureblob, terrablobgit
 # Defaults to memory
 # Env override: ATHENS_STORAGE_TYPE
-StorageType = "terrablob"
+StorageType = "s3"
 
 # Certificate and key to make athens serve using https instead of plain text http.
 # Set both to enable.
@@ -408,27 +408,27 @@ SingleFlightType = "memory"
 
         # Region for S3 storage
         # Env override: AWS_REGION
-        Region = "MY_AWS_REGION"
+        Region = "us-west-2"
 
         # Access Key for S3 storage
         # Env override: AWS_ACCESS_KEY_ID
-        Key = "MY_AWS_ACCESS_KEY_ID"
+        Key = "goproxy"
 
         # Secret Key for S3 storage
         # Env override: AWS_SECRET_ACCESS_KEY
-        Secret = "MY_AWS_SECRET_ACCESS_KEY"
+        Secret = "goproxy"
 
         # Session Token for S3 storage
         # Env override: AWS_SESSION_TOKEN
         Token = ""
 
         # S3 Bucket to use for storage
         # Env override: ATHENS_S3_BUCKET_NAME
-        Bucket = "MY_S3_BUCKET_NAME"
+        Bucket = "/"
 
         # If true then path style url for s3 endpoint will be used
         # Env override: AWS_FORCE_PATH_STYLE
-        ForcePathStyle = false
+        ForcePathStyle = true
 
         # If true then the default aws configuration will be used. This will
         # attempt to find credentials in the environment, in the shared
@@ -454,7 +454,8 @@ SingleFlightType = "memory"
         #
         # You must still provide a `Region` value when specifying an endpoint.
         # Env override: AWS_ENDPOINT
-        Endpoint = ""
+        # Tier 3 port: https://engwiki.uberinternal.com/display/TERRABLOB/S3+HTTP+API
+        Endpoint = "http://localhost:18839"
 
    [Storage.AzureBlob]
         # Storage Account name for Azure Blob