gensupport: expand retryable errors (#529)

* gensupport: expand retryable errors

This commit improves retry handling by allowing retries
for the following:
* Wrapped errors that are retryable (Go 1.13+ only)
* Transient network errors

Following retry guidance from
https://cloud.google.com/storage/docs/exponential-backoff

Note that this conflicts with #528, but it's an easy conflict
for me to fix once one of them is merged.

Fixes #449

internal/gensupport/resumable.go

@@ -28,6 +28,8 @@ var (
 	backoff       = func() Backoff {
 		return &gax.Backoff{Initial: 100 * time.Millisecond}
 	}
+	// isRetryable is a platform-specific hook, specified in retryable_linux.go
+	syscallRetryable func(error) bool = func(err error) bool { return false }
 )
 
 const (
@@ -226,7 +228,8 @@ func (rx *ResumableUpload) Upload(ctx context.Context) (resp *http.Response, err
 }
 
 // shouldRetry indicates whether an error is retryable for the purposes of this
-// package.
+// package, following guidance from
+// https://cloud.google.com/storage/docs/exponential-backoff .
 func shouldRetry(status int, err error) bool {
 	if 500 <= status && status <= 599 {
 		return true
@@ -237,8 +240,19 @@ func shouldRetry(status int, err error) bool {
 	if err == io.ErrUnexpectedEOF {
 		return true
 	}
+	// Transient network errors should be retried.
+	if syscallRetryable(err) {
+		return true
+	}
 	if err, ok := err.(interface{ Temporary() bool }); ok {
-		return err.Temporary()
+		if err.Temporary() {
+			return true
+		}
+	}
+	// If Go 1.13 error unwrapping is available, use this to examine wrapped
+	// errors.
+	if err, ok := err.(interface{ Unwrap() error }); ok {
+		return shouldRetry(status, err.Unwrap())
 	}
 	return false
 }

internal/gensupport/retryable_linux.go

@@ -0,0 +1,15 @@
+// Copyright 2020 Google LLC.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux
+
+package gensupport
+
+import "syscall"
+
+func init() {
+	// Initialize syscallRetryable to return true on transient socket-level
+	// errors. These errors are specific to Linux.
+	syscallRetryable = func(err error) bool { return err == syscall.ECONNRESET || err == syscall.ECONNREFUSED }
+}