Merge release/0.9.8

README.md

@@ -145,6 +145,8 @@ res, err := request.Send(&request.Options{
 
 By default, upon receiving a retryable status code, `Send` will use am exponential backoff algorithm to retry the request. By default, it will wait for 3 seconds before retrying for 5 minutes, then 9 seconds between 5 and 10 minutes, then 27 seconds between 10 and 15 minutes, etc.
 
+Connection errors like EOF, connection reset, etc. are also retried `Options.Attempts` times. The default is 5 attempts and the code waits for `Options.InterAttemptDelay` (Default: 3s).
+
 You can change the delay and the backoff factor like this:
 
 ```go

request.go

@@ -9,13 +9,15 @@ import (
 	"math"
 	"mime"
 	"mime/multipart"
+	"net"
 	"net/http"
 	"net/textproto"
 	"net/url"
 	"path/filepath"
 	"reflect"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
 	"github.com/gildas/go-core"
@@ -109,14 +111,25 @@ func Send(options *Options, results interface{}) (*Content, error) {
 	// Sending the request...
 	start := time.Now()
 	for attempt := uint(0); attempt < options.Attempts; attempt++ {
-		log.Tracef("Attempt #%d/%d", attempt+1, options.Attempts)
+		log.Tracef("Attempt #%d/%d (timeout: %s)", attempt+1, options.Attempts, httpclient.Timeout)
 		req.Header.Set("X-Attempt", strconv.FormatUint(uint64(attempt+1), 10))
 		log.Tracef("Request Headers: %#v", req.Header)
 		reqStart := time.Now()
 		res, err := httpclient.Do(req)
 		reqDuration := time.Since(reqStart)
 		log = log.Record("duration", reqDuration/time.Millisecond)
 		if err != nil {
+			netErr := &net.OpError{}
+			if errors.As(err, &netErr) && errors.Is(netErr, syscall.ECONNRESET) {
+				if attempt+1 < options.Attempts {
+					log.Warnf("Temporary failed to send request (duration: %s/%s), Error: %s", reqDuration, options.Timeout, err.Error()) // we don't want the stack here
+					log.Infof("Waiting for %s before trying again", options.InterAttemptDelay)
+					time.Sleep(options.InterAttemptDelay)
+					req, _ = buildRequest(log, options)
+					continue
+				}
+				break
+			}
 			urlErr := &url.Error{}
 			if errors.As(err, &urlErr) {
 				if urlErr.Timeout() || urlErr.Temporary() || urlErr.Unwrap() == io.EOF || errors.Is(err, context.DeadlineExceeded) {

request_test.go

@@ -702,6 +702,32 @@ func (suite *RequestSuite) TestCanRetryReceivingRequest() {
 	suite.Require().NoError(err, "Failed reading response content, err=%+v", err)
 }
 
+func (suite *RequestSuite) TestCanRetryReceivingRequestECONNRESET() {
+	server := CreateEConnResetTestServer(suite, 3)
+	serverURL, _ := url.Parse(server.URL)
+	_, err := request.Send(&request.Options{
+		URL:                  serverURL,
+		RetryableStatusCodes: []int{http.StatusServiceUnavailable},
+		Attempts:             5,
+		Timeout:              1 * time.Second,
+		Logger:               suite.Logger,
+	}, nil)
+	suite.Require().NoError(err, "Failed reading response content, err=%+v", err)
+}
+
+func (suite *RequestSuite) TestCanRetryReceivingRequestEOF() {
+	serverURL, _ := url.Parse(suite.Server.URL)
+	serverURL, _ = serverURL.Parse("/retry_eof")
+	_, err := request.Send(&request.Options{
+		URL:                  serverURL,
+		RetryableStatusCodes: []int{http.StatusServiceUnavailable},
+		Attempts:             5,
+		Timeout:              1 * time.Second,
+		Logger:               suite.Logger,
+	}, nil)
+	suite.Require().NoError(err, "Failed reading response content, err=%+v", err)
+}
+
 func (suite *RequestSuite) TestCanRetryReceivingRequestWithExponentialBackoff() {
 	start := time.Now()
 	serverURL, _ := url.Parse(suite.Server.URL)
@@ -759,7 +785,7 @@ func (suite *RequestSuite) TestCanRetryReceivingRequestWithRetryAfter() {
 	duration := time.Since(start)
 	suite.Require().NoError(err, "Failed reading response content, err=%+v", err)
 	suite.Assert().GreaterOrEqual(int64(duration), int64(4*time.Second), "The request lasted less than 4 second (%s)", duration)
-	suite.Assert().Less(int64(duration), int64(5*time.Second), "The request lasted more than 5 second (%s)", duration)
+	suite.Assert().Less(int64(duration), int64(7*time.Second), "The request lasted more than 7 second (%s)", duration)
 }
 
 func (suite *RequestSuite) TestCanRetryPostingRequest() {
@@ -964,6 +990,20 @@ func (suite *RequestSuite) TestShouldFailReceivingBadResponse() {
 	suite.Assert().Contains(err.Error(), "unexpected EOF")
 }
 
+func (suite *RequestSuite) TestShouldFailRetryingECONNRESETTooManyTimes() {
+	server := CreateEConnResetTestServer(suite, 10)
+	serverURL, _ := url.Parse(server.URL)
+	_, err := request.Send(&request.Options{
+		URL:                  serverURL,
+		RetryableStatusCodes: []int{http.StatusServiceUnavailable},
+		Attempts:             3,
+		Timeout:              1 * time.Second,
+		Logger:               suite.Logger,
+	}, nil)
+	suite.Require().Error(err, "Should have failed sending request")
+	suite.Logger.Errorf("Expected Error", err)
+}
+
 func (suite *RequestSuite) TestCanSendPostRequestWithRedirect() {
 	serverURL, _ := url.Parse(suite.Server.URL)
 	serverURL, _ = serverURL.Parse("/redirect")

server_test.go

@@ -3,19 +3,22 @@ package request_test
 import (
 	"encoding/json"
 	"fmt"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"strconv"
 	"strings"
+	"sync"
+	"syscall"
 	"time"
 
 	"github.com/gildas/go-core"
 	"github.com/gildas/go-errors"
 	"github.com/gildas/go-request"
 )
 
-func CreateTestServer(suite *RequestSuite) *httptest.Server {
-	return httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {
+func CreateTestServerHandler(suite *RequestSuite, server *httptest.Server) http.HandlerFunc {
+	return http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {
 		log := suite.Logger.Child("server", "handler")
 		headers := map[string]string{}
 		for key, values := range req.Header {
@@ -263,6 +266,14 @@ func CreateTestServer(suite *RequestSuite) *httptest.Server {
 				if _, err := res.Write([]byte(`body`)); err != nil {
 					log.Errorf("Failed to Write response to %s %s, error: %s", req.Method, req.URL, err)
 				}
+			case "/retry_eof":
+				max := core.Atoi(req.Header.Get("X-Max-Retry"), 5)
+				attempt := core.Atoi(req.Header.Get("X-Attempt"), 0)
+				if attempt < max { // On the max-th attempt, we want to return 200
+					log.Infof("Disconnecting client connections")
+					server.CloseClientConnections()
+					return
+				}
 			case "/text_data":
 				reqAccept := req.Header.Get("Accept")
 				log.Infof("Request Accept: %s", reqAccept)
@@ -366,6 +377,54 @@ func CreateTestServer(suite *RequestSuite) *httptest.Server {
 			}
 			return
 		}
-	}))
+	})
+}
+
+func CreateTestServer(suite *RequestSuite) *httptest.Server {
+	testserver := httptest.NewUnstartedServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {}))
+	testserver.Config = &http.Server{Handler: CreateTestServerHandler(suite, testserver)}
+	testserver.Start()
+	return testserver
+}
+
+type EConnResetListener struct {
+	net.Listener
+	MaxResets int
+	attempts  int
+	lock      sync.Mutex
+	Suite     *RequestSuite
+}
+
+func (listener *EConnResetListener) Accept() (net.Conn, error) {
+	log := listener.Suite.Logger.Child("listener", "accept")
+	conn, err := listener.Listener.Accept()
+	if err != nil {
+		return nil, err
+	}
+	listener.lock.Lock()
+	defer listener.lock.Unlock()
+
+	if listener.attempts < listener.MaxResets {
+		log.Infof("Connection Reset #%d", listener.attempts)
+		listener.attempts++
+		conn.Close()
+		return nil, &net.OpError{Op: "accept", Net: "tcp", Source: nil, Addr: conn.RemoteAddr(), Err: syscall.ECONNRESET}
+	}
+	return conn, nil
+}
 
+func CreateEConnResetTestServer(suite *RequestSuite, maxResets int) *httptest.Server {
+	listener := &EConnResetListener{
+		Listener:  core.Must(net.Listen("tcp", "127.0.0.1:0")),
+		MaxResets: maxResets,
+		attempts:  0,
+		Suite:     suite,
+	}
+	testserver := &httptest.Server{
+		Listener: listener,
+		Config:   &http.Server{Handler: http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {})},
+	}
+	testserver.Config = &http.Server{Handler: CreateTestServerHandler(suite, testserver)}
+	testserver.Start()
+	return testserver
 }

version.go

@@ -4,4 +4,4 @@ package request
 var commit string
 
 // VERSION is the version of this library
-var VERSION = "0.9.7" + commit
+var VERSION = "0.9.8" + commit