diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3b2b4ee20..deceff891 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -141,7 +141,7 @@ jobs: - name: pipelines tests run: go test -v github.com/jfrog/jfrog-client-go/tests --timeout 0 --test.pipelines --rt.url=${{ secrets.PLATFORM_URL }}/artifactory --pipe.url=${{ secrets.PLATFORM_URL }}/pipelines --rt.user=${{ secrets.PLATFORM_USER }} --rt.password=${{ secrets.PLATFORM_PASSWORD }} --pipe.accessToken=${{ secrets.PLATFORM_ADMIN_TOKEN }} --pipe.vcsToken=${{ secrets.CLI_PIPE_VCS_TOKEN }} --pipe.vcsRepo=${{ secrets.CLI_PIPE_VCS_REPO }} --pipe.vcsBranch=${{ secrets.CLI_PIPE_VCS_BRANCH }} --ci.runId=${{ runner.os }}_pipe - + JFrog-Client-Go-Repositories-Tests: needs: Pretest name: repositories ubuntu-latest diff --git a/artifactory/services/download.go b/artifactory/services/download.go index 4a2d73ed9..14f9c7f43 100644 --- a/artifactory/services/download.go +++ b/artifactory/services/download.go @@ -89,7 +89,7 @@ func (ds *DownloadService) getOperationSummary(totalSucceeded, totalFailed int) return operationSummary } -func (ds *DownloadService) DownloadFiles(downloadParams ...DownloadParams) (opertaionSummary *utils.OperationSummary, err error) { +func (ds *DownloadService) DownloadFiles(downloadParams ...DownloadParams) (operationSummary *utils.OperationSummary, err error) { producerConsumer := parallel.NewRunner(ds.GetThreads(), 20000, false) errorsQueue := clientutils.NewErrorsQueue(1) expectedChan := make(chan int, 1) @@ -117,7 +117,7 @@ func (ds *DownloadService) DownloadFiles(downloadParams ...DownloadParams) (oper for _, v := range successCounters { totalSuccess += v } - opertaionSummary = ds.getOperationSummary(totalSuccess, <-expectedChan-totalSuccess) + operationSummary = ds.getOperationSummary(totalSuccess, <-expectedChan-totalSuccess) return } @@ -236,7 +236,7 @@ func (ds *DownloadService) produceTasks(reader *content.ContentReader, downloadP return tasksCount } defer func() { - if err := sortedReader.Close(); err != nil { + if err = sortedReader.Close(); err != nil { log.Warn("Could not close sortedReader. Error: " + err.Error()) } }() @@ -287,7 +287,7 @@ func rbGpgValidate(rbGpgValidationMap map[string]*utils.RbGpgValidator, bundle s } // Extract for the aqlResultItem the directory path, store the path the directoriesDataKeys and in the directoriesData map. -// In addition directoriesData holds the correlate DownloadData for each key, later on this DownloadData will be used to create a create dir tasks if needed. +// In addition, directoriesData holds the correlate DownloadData for each key, later on this DownloadData will be used to create a create dir tasks if needed. // This function append the new data to directoriesDataKeys and to directoriesData and return the new map and the new []string // We are storing all the keys of directoriesData in additional array(directoriesDataKeys) so we could sort the keys and access the maps in the sorted order. func collectDirPathsToCreate(aqlResultItem utils.ResultItem, directoriesData map[string]DownloadData, tempData DownloadData, directoriesDataKeys []string) (map[string]DownloadData, []string) { @@ -455,7 +455,7 @@ func createLocalSymlink(localPath, localFileName, symlinkArtifact string, symlin } // We can't create symlink in case a file with the same name already exist, we must remove the file before creating the symlink if isFileExists { - if err := os.Remove(localSymlinkPath); err != nil { + if err = os.Remove(localSymlinkPath); err != nil { return err } } @@ -495,34 +495,34 @@ func (ds *DownloadService) createFileHandlerFunc(downloadParams DownloadParams, return func(downloadData DownloadData) parallel.TaskFunc { return func(threadId int) error { logMsgPrefix := clientutils.GetLogMsgPrefix(threadId, ds.DryRun) - downloadPath, e := clientutils.BuildUrl(ds.GetArtifactoryDetails().GetUrl(), downloadData.Dependency.GetItemRelativePath(), make(map[string]string)) - if e != nil { - return e + downloadPath, err := clientutils.BuildUrl(ds.GetArtifactoryDetails().GetUrl(), downloadData.Dependency.GetItemRelativePath(), make(map[string]string)) + if err != nil { + return err } log.Info(logMsgPrefix+"Downloading", downloadData.Dependency.GetItemRelativePath()) if ds.DryRun { successCounters[threadId]++ return nil } - target, placeholdersUsed, e := clientutils.BuildTargetPath(downloadData.DownloadPath, downloadData.Dependency.GetItemRelativePath(), downloadData.Target, true) - if e != nil { - return e + target, placeholdersUsed, err := clientutils.BuildTargetPath(downloadData.DownloadPath, downloadData.Dependency.GetItemRelativePath(), downloadData.Target, true) + if err != nil { + return err } localPath, localFileName := fileutils.GetLocalPathAndFile(downloadData.Dependency.Name, downloadData.Dependency.Path, target, downloadData.Flat, placeholdersUsed) if downloadData.Dependency.Type == "folder" { return createDir(localPath, localFileName, logMsgPrefix) } - if e = removeIfSymlink(filepath.Join(localPath, localFileName)); e != nil { - return e + if err = removeIfSymlink(filepath.Join(localPath, localFileName)); err != nil { + return err } if downloadParams.IsSymlink() { if isSymlink, e := ds.createSymlinkIfNeeded(ds.GetArtifactoryDetails().GetUrl(), localPath, localFileName, logMsgPrefix, downloadData, successCounters, threadId, downloadParams); isSymlink { return e } } - if e = ds.downloadFileIfNeeded(downloadPath, localPath, localFileName, logMsgPrefix, downloadData, downloadParams); e != nil { - log.Error(logMsgPrefix, "Received an error: "+e.Error()) - return e + if err = ds.downloadFileIfNeeded(downloadPath, localPath, localFileName, logMsgPrefix, downloadData, downloadParams); err != nil { + log.Error(logMsgPrefix, "Received an error: "+err.Error()) + return err } successCounters[threadId]++ ds.addToResults(&downloadData.Dependency, ds.GetArtifactoryDetails().GetUrl(), localPath, localFileName) @@ -532,9 +532,9 @@ func (ds *DownloadService) createFileHandlerFunc(downloadParams DownloadParams, } func (ds *DownloadService) downloadFileIfNeeded(downloadPath, localPath, localFileName, logMsgPrefix string, downloadData DownloadData, downloadParams DownloadParams) error { - isEqual, e := fileutils.IsEqualToLocalFile(filepath.Join(localPath, localFileName), downloadData.Dependency.Actual_Md5, downloadData.Dependency.Actual_Sha1) - if e != nil { - return e + isEqual, err := fileutils.IsEqualToLocalFile(filepath.Join(localPath, localFileName), downloadData.Dependency.Actual_Md5, downloadData.Dependency.Actual_Sha1) + if err != nil { + return err } if isEqual { log.Debug(logMsgPrefix + "File already exists locally.") @@ -542,9 +542,9 @@ func (ds *DownloadService) downloadFileIfNeeded(downloadPath, localPath, localFi ds.Progress.IncrementGeneralProgress() } if downloadParams.IsExplode() { - e = clientutils.ExtractArchive(localPath, localFileName, downloadData.Dependency.Name, logMsgPrefix, downloadParams.IsBypassArchiveInspection()) + err = clientutils.ExtractArchive(localPath, localFileName, downloadData.Dependency.Name, logMsgPrefix, downloadParams.IsBypassArchiveInspection()) } - return e + return err } downloadFileDetails := createDownloadFileDetails(downloadPath, localPath, localFileName, downloadData, downloadParams.IsSkipChecksum()) return ds.downloadFile(downloadFileDetails, logMsgPrefix, downloadParams) @@ -552,8 +552,8 @@ func (ds *DownloadService) downloadFileIfNeeded(downloadPath, localPath, localFi func createDir(localPath, localFileName, logMsgPrefix string) error { folderPath := filepath.Join(localPath, localFileName) - if e := fileutils.CreateDirIfNotExist(folderPath); e != nil { - return e + if err := fileutils.CreateDirIfNotExist(folderPath); err != nil { + return err } log.Info(logMsgPrefix + "Creating folder: " + folderPath) return nil @@ -564,8 +564,8 @@ func (ds *DownloadService) createSymlinkIfNeeded(rtUrl, localPath, localFileName isSymlink := len(symlinkArtifact) > 0 if isSymlink { symlinkChecksum := getArtifactSymlinkChecksum(downloadData.Dependency.Properties) - if e := createLocalSymlink(localPath, localFileName, symlinkArtifact, downloadParams.ValidateSymlinks(), symlinkChecksum, logMsgPrefix); e != nil { - return isSymlink, e + if err := createLocalSymlink(localPath, localFileName, symlinkArtifact, downloadParams.ValidateSymlinks(), symlinkChecksum, logMsgPrefix); err != nil { + return isSymlink, err } successCounters[threadId]++ ds.addToResults(&downloadData.Dependency, rtUrl, localPath, localFileName) diff --git a/artifactory/services/upload.go b/artifactory/services/upload.go index 53fefb4a8..d27f64182 100644 --- a/artifactory/services/upload.go +++ b/artifactory/services/upload.go @@ -35,7 +35,7 @@ const ( DefaultMinChecksumDeploy = utils.SizeKib * 10 // The default minimum file size for attempting multi-part upload defaultUploadMinSplit = utils.SizeMiB * 200 - // The default maximum number of parts that can be concurrently uploaded per file during a multi-part upload + // The default maximum number of parts that can be concurrently uploaded per file during a multipart upload defaultUploadSplitCount = 5 ) diff --git a/artifactory/services/utils/specutils.go b/artifactory/services/utils/specutils.go index f24eab695..45f82ce24 100644 --- a/artifactory/services/utils/specutils.go +++ b/artifactory/services/utils/specutils.go @@ -50,30 +50,6 @@ type CommonParams struct { Include []string } -type FileGetter interface { - GetAql() Aql - GetPattern() string - SetPattern(pattern string) - GetExclusions() []string - GetTarget() string - SetTarget(target string) - IsExplode() bool - GetProps() string - GetSortOrder() string - GetSortBy() []string - GetOffset() int - GetLimit() int - GetBuild() string - GetProject() string - GetBundle() string - GetSpecType() (specType SpecType) - IsRecursive() bool - IsIncludeDirs() bool - GetArchiveEntries() string - SetArchiveEntries(archiveEntries string) - GetPatternType() clientutils.PatternType -} - func (params CommonParams) GetArchiveEntries() string { return params.ArchiveEntries } @@ -110,10 +86,6 @@ func (params *CommonParams) GetExcludeProps() string { return params.ExcludeProps } -func (params *CommonParams) IsExplode() bool { - return params.Recursive -} - func (params *CommonParams) IsRecursive() bool { return params.Recursive } diff --git a/go.mod b/go.mod index f2fb1aafa..d78112afa 100644 --- a/go.mod +++ b/go.mod @@ -9,14 +9,14 @@ require ( github.com/go-git/go-git/v5 v5.12.0 github.com/golang-jwt/jwt/v4 v4.5.0 github.com/gookit/color v1.5.4 - github.com/jfrog/archiver/v3 v3.6.0 + github.com/jfrog/archiver/v3 v3.6.1 github.com/jfrog/build-info-go v1.9.29 - github.com/jfrog/gofrog v1.7.2 + github.com/jfrog/gofrog v1.7.3 github.com/stretchr/testify v1.9.0 github.com/xanzy/ssh-agent v0.3.3 - golang.org/x/crypto v0.23.0 - golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 - golang.org/x/term v0.20.0 + golang.org/x/crypto v0.25.0 + golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 + golang.org/x/term v0.22.0 ) require ( @@ -35,7 +35,7 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/klauspost/compress v1.17.4 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/klauspost/cpuid/v2 v2.2.3 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/minio/sha256-simd v1.0.1 // indirect @@ -46,18 +46,18 @@ require ( github.com/rivo/uniseg v0.4.7 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/skeema/knownhosts v1.2.2 // indirect - github.com/ulikunitz/xz v0.5.11 // indirect + github.com/ulikunitz/xz v0.5.12 // indirect github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect - golang.org/x/mod v0.17.0 // indirect - golang.org/x/net v0.25.0 // indirect + golang.org/x/mod v0.18.0 // indirect + golang.org/x/net v0.26.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.20.0 // indirect - golang.org/x/tools v0.21.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/tools v0.22.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) // replace github.com/jfrog/build-info-go => github.com/eyalbe4/build-info-go v1.8.6-0.20240610015232-844595d5a4f3 -// replace github.com/jfrog/gofrog => github.com/jfrog/gofrog dev +// replace github.com/jfrog/gofrog => github.com/jfrog/gofrog dev \ No newline at end of file diff --git a/go.sum b/go.sum index 1f17331fa..b3cf6ebcc 100644 --- a/go.sum +++ b/go.sum @@ -57,17 +57,17 @@ github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0= github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/jfrog/archiver/v3 v3.6.0 h1:OVZ50vudkIQmKMgA8mmFF9S0gA47lcag22N13iV3F1w= -github.com/jfrog/archiver/v3 v3.6.0/go.mod h1:fCAof46C3rAXgZurS8kNRNdSVMKBbZs+bNNhPYxLldI= +github.com/jfrog/archiver/v3 v3.6.1 h1:LOxnkw9pOn45DzCbZNFV6K0+6dCsQ0L8mR3ZcujO5eI= +github.com/jfrog/archiver/v3 v3.6.1/go.mod h1:VgR+3WZS4N+i9FaDwLZbq+jeU4B4zctXL+gL4EMzfLw= github.com/jfrog/build-info-go v1.9.29 h1:3vJ+kbk9PpU6wjisXi9c4qISNpYkISh/NmB5mq1ZlSY= github.com/jfrog/build-info-go v1.9.29/go.mod h1:AzFJlN/yKfKuKcSBaGy5nNmKN1xzx6+XcRWAswCTLTA= -github.com/jfrog/gofrog v1.7.2 h1:VkAaA/9tmbw27IqgUOmaZWnO6ATUqL3vRzDnsROKATw= -github.com/jfrog/gofrog v1.7.2/go.mod h1:WJFk88SR9Sr9mKl1bQBig7DmSdXiBGKV3WhL9O6jL9w= +github.com/jfrog/gofrog v1.7.3 h1:34iaAZP9qY1dkjb8a0g0jn0u9/2k8RROx4hgnZNTAQw= +github.com/jfrog/gofrog v1.7.3/go.mod h1:4MH6RLH0XF96Y3PK7Cy9u8YvxN9cbe0VJHlzEfMpJDA= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= -github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= @@ -111,8 +111,8 @@ github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8 github.com/terminalstatic/go-xsd-validate v0.1.5 h1:RqpJnf6HGE2CB/lZB1A8BYguk8uRtcvYAPLCF15qguo= github.com/terminalstatic/go-xsd-validate v0.1.5/go.mod h1:18lsvYFofBflqCrvo1umpABZ99+GneNTw2kEEc8UPJw= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= -github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= -github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= +github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= @@ -131,14 +131,14 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= -golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= -golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= -golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= +golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -146,8 +146,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= -golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -167,15 +167,15 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= -golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= +golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= -golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= -golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= +golang.org/x/term v0.22.0/go.mod h1:F3qCibpT5AMpCRfhfT53vVJwhLtIVHhB9XDjfFvnMI4= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -183,14 +183,14 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= -golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= +golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/http/httpclient/client.go b/http/httpclient/client.go index 4aa336eb9..687f80093 100644 --- a/http/httpclient/client.go +++ b/http/httpclient/client.go @@ -179,16 +179,18 @@ func (jc *HttpClient) doRequest(req *http.Request, content []byte, followRedirec copyHeaders(httpClientsDetails, req) addUberTraceIdHeaderIfSet(req) + client := jc.client + if !followRedirect || (followRedirect && req.Method == http.MethodPost) { - jc.client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + // The jc.client is a shared resource between go routines, so to handle this override we clone it. + client = cloneHttpClient(jc.client) + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { redirectUrl = req.URL.String() return errors.New("redirect") } } - resp, err = jc.client.Do(req) - jc.client.CheckRedirect = nil - + resp, err = client.Do(req) if err != nil && redirectUrl != "" { if !followRedirect { log.Debug("Blocking HTTP redirect to", redirectUrl) @@ -203,9 +205,7 @@ func (jc *HttpClient) doRequest(req *http.Request, content []byte, followRedirec return } } - - err = errorutils.CheckError(err) - if err != nil { + if errorutils.CheckError(err) != nil { return } if closeBody { @@ -219,6 +219,15 @@ func (jc *HttpClient) doRequest(req *http.Request, content []byte, followRedirec return } +func cloneHttpClient(httpClient *http.Client) *http.Client { + return &http.Client{ + Transport: httpClient.Transport, + Timeout: httpClient.Timeout, + Jar: httpClient.Jar, + CheckRedirect: httpClient.CheckRedirect, + } +} + func copyHeaders(httpClientsDetails httputils.HttpClientDetails, req *http.Request) { if httpClientsDetails.Headers != nil { for name := range httpClientsDetails.Headers { @@ -475,7 +484,7 @@ func saveToFile(downloadFileDetails *DownloadFileDetails, resp *http.Response, p } if hex.EncodeToString(actualSha1.Sum(nil)) != downloadFileDetails.ExpectedSha1 { - err = errors.New("Checksum mismatch for " + fileName + ", expected: " + downloadFileDetails.ExpectedSha1 + ", actual: " + hex.EncodeToString(actualSha1.Sum(nil))) + err = errors.New("checksum mismatch for " + fileName + ", expected: " + downloadFileDetails.ExpectedSha1 + ", actual: " + hex.EncodeToString(actualSha1.Sum(nil))) } } else { _, err = io.Copy(out, reader) @@ -660,21 +669,31 @@ func mergeChunks(chunksPaths []string, flags ConcurrentDownloadFlags) (err error writer = io.MultiWriter(destFile) } for i := 0; i < flags.SplitCount; i++ { - reader, err := os.Open(chunksPaths[i]) - if err != nil { - return err - } - defer func() { - err = errors.Join(err, errorutils.CheckError(reader.Close())) + // Wrapping the loop body in an anonymous function to ensure deferred calls + // are executed at the end of each iteration, not at the end of the enclosing function. + err = func() (e error) { + reader, e := os.Open(chunksPaths[i]) + if errorutils.CheckError(e) != nil { + return e + } + defer func() { + e = errors.Join(e, errorutils.CheckError(reader.Close())) + }() + + _, e = io.Copy(writer, reader) + if errorutils.CheckError(e) != nil { + return e + } + + return nil }() - _, err = io.Copy(writer, reader) if err != nil { return err } } if len(flags.ExpectedSha1) > 0 && !flags.SkipChecksum { if hex.EncodeToString(actualSha1.Sum(nil)) != flags.ExpectedSha1 { - err = errors.New("Checksum mismatch for " + flags.LocalFileName + ", expected: " + flags.ExpectedSha1 + ", actual: " + hex.EncodeToString(actualSha1.Sum(nil))) + err = errorutils.CheckErrorf("checksum mismatch for " + flags.LocalFileName + ", expected: " + flags.ExpectedSha1 + ", actual: " + hex.EncodeToString(actualSha1.Sum(nil))) } } return err diff --git a/http/httpclient/retryableconnection.go b/http/httpclient/retryableconnection.go index a73c67fe7..083740f84 100644 --- a/http/httpclient/retryableconnection.go +++ b/http/httpclient/retryableconnection.go @@ -105,10 +105,7 @@ func (m *monitor) start() (result []byte, stable bool, err error) { } defer func() { if m.resp != nil && m.resp.Body != nil { - e := m.resp.Body.Close() - if err == nil { - err = errorutils.CheckError(e) - } + err = errors.Join(err, m.resp.Body.Close()) } }() @@ -117,7 +114,8 @@ func (m *monitor) start() (result []byte, stable bool, err error) { bodyReader := bufio.NewReader(m.resp.Body) go func() { for { - line, _, err := bodyReader.ReadLine() + var line []byte + line, _, err = bodyReader.ReadLine() if err == io.EOF { errChan <- nil break diff --git a/utils/io/content/contentreader.go b/utils/io/content/contentreader.go index 465f6e271..f1737dfa4 100644 --- a/utils/io/content/contentreader.go +++ b/utils/io/content/contentreader.go @@ -163,7 +163,7 @@ func (cr *ContentReader) readSingleFile(filePath string) { } for dec.More() { var ResultItem map[string]interface{} - err := dec.Decode(&ResultItem) + err = dec.Decode(&ResultItem) if err != nil { log.Error(err) cr.errorsQueue.AddError(errorutils.CheckError(err)) @@ -397,7 +397,7 @@ func MergeSortedReaders(readerRecord SortableContentItem, sortedReaders []*Conte for i := 0; i < len(sortedFilesClone); i++ { if currentContentItem[i] == nil && sortedFilesClone[i] != nil { temp := (reflect.New(valueType)).Interface() - if err := sortedFilesClone[i].NextRecord(temp); nil != err { + if err = sortedFilesClone[i].NextRecord(temp); nil != err { sortedFilesClone[i] = nil continue } diff --git a/utils/io/content/contentwriter.go b/utils/io/content/contentwriter.go index 3e77feced..0dbbb89f1 100644 --- a/utils/io/content/contentwriter.go +++ b/utils/io/content/contentwriter.go @@ -23,7 +23,7 @@ const ( // Write a JSON file in small chunks. Only a single JSON key can be written to the file, and array as its value. // The array's values could be any JSON value types (number, string, etc...). -// Once the first 'Write" call is made, the file will stay open, waiting for the next struct to be written (thread-safe). +// Once the first 'Write' call is made, the file will stay open, waiting for the next struct to be written (thread-safe). // Finally, 'Close' will fill the end of the JSON file and the operation will be completed. type ContentWriter struct { // arrayKey = JSON object key to be written. diff --git a/utils/tests/utils.go b/utils/tests/utils.go index 8f3993073..99583ead6 100644 --- a/utils/tests/utils.go +++ b/utils/tests/utils.go @@ -84,7 +84,7 @@ func RunTests(testsPackages []string, hideUnitTestsLog bool) error { exitOnErr(err) cmd.Stdout, cmd.Stderr = f, f - if err := cmd.Run(); err != nil { + if err = cmd.Run(); err != nil { log.Error("Unit tests failed, full report available at the following path:", f.Name()) exitOnErr(err) }