From 2a4300add919b1d54ba41c8999d5ab5344b50d93 Mon Sep 17 00:00:00 2001 From: Andy Trimble Date: Wed, 11 Oct 2017 18:34:00 -0600 Subject: [PATCH 1/6] Added optional rate limit handling. --- sendgrid.go | 48 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/sendgrid.go b/sendgrid.go index 8d5b02a7..bdbced9b 100644 --- a/sendgrid.go +++ b/sendgrid.go @@ -1,10 +1,20 @@ // Package sendgrid provides a simple interface to interact with the SendGrid API package sendgrid -import "github.com/sendgrid/rest" // depends on version 2.2.0 +import ( + "errors" + "net/http" + "time" + + "github.com/sendgrid/rest" +) // depends on version 2.2.0 // Version is this client library's current version -const Version = "3.1.0" +const ( + Version = "3.1.0" + rateLimitRetry = 5 + rateLimitSleep = 1100 +) // GetRequest returns a default request object. func GetRequest(key string, endpoint string, host string) rest.Request { @@ -30,3 +40,37 @@ var DefaultClient = rest.DefaultClient func API(request rest.Request) (*rest.Response, error) { return DefaultClient.API(request) } + +// Request attempts a request asynchronously in a new go +// routine. This function returns two channels: responses +// and errors. This function will retry in the case of a +// rate limit. +func Request(request rest.Request) (chan *rest.Response, chan error) { + r := make(chan *rest.Response) + e := make(chan error) + + go func() { + retry := 0 + for { + response, err := DefaultClient.API(request) + if err != nil { + e <- err + return + } + + if response.StatusCode == http.StatusTooManyRequests { + if retry > rateLimitRetry { + e <- errors.New("Rate limit retry exceeded") + return + } + retry++ + time.Sleep(rateLimitSleep * time.Millisecond) + continue + } + + r <- response + } + }() + + return r, e +} From 42aa4f208e7687a9a30b1b57684c6012d53ff151 Mon Sep 17 00:00:00 2001 From: Andy Trimble Date: Thu, 12 Oct 2017 14:08:31 -0600 Subject: [PATCH 2/6] Addressed comments, fixed a bug. --- sendgrid.go | 70 ++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 50 insertions(+), 20 deletions(-) diff --git a/sendgrid.go b/sendgrid.go index 6ffd1cab..e7baafd7 100644 --- a/sendgrid.go +++ b/sendgrid.go @@ -4,6 +4,7 @@ package sendgrid import ( "errors" "net/http" + "strconv" "time" "github.com/sendgrid/rest" // depends on version 2.2.0 @@ -56,37 +57,66 @@ func NewSendClient(key string) *Client { var DefaultClient = rest.DefaultClient // API sets up the request to the SendGrid API, this is main interface. +// This function is deprecated. Please use the MakeRequest or +// MakeRequestAsync functions. func API(request rest.Request) (*rest.Response, error) { return DefaultClient.API(request) } -// Request attempts a request asynchronously in a new go +// MakeRequest attemps a SendGrid request synchronously. +func MakeRequest(request rest.Request) (*rest.Response, error) { + return DefaultClient.API(request) +} + +// MakeRequestRetry a synchronous request, but retry in the event of a rate +// limited response. +func MakeRequestRetry(request rest.Request) (*rest.Response, error) { + retry := 0 + var response *rest.Response + var err error + + for { + response, err = DefaultClient.API(request) + if err != nil { + return nil, err + } + + if response.StatusCode != http.StatusTooManyRequests { + return response, nil + } + + if retry > rateLimitRetry { + return nil, errors.New("Rate limit retry exceeded") + } + retry++ + + resetTime := time.Now().Add(rateLimitSleep * time.Millisecond) + + reset, ok := response.Headers["X-RateLimit-Reset"] + if ok && len(reset) > 0 { + t, err := strconv.Atoi(reset[0]) + if err == nil { + resetTime = time.Unix(int64(t), 0) + } + } + time.Sleep(time.Until(resetTime)) + } +} + +// MakeRequestAsync attempts a request asynchronously in a new go // routine. This function returns two channels: responses // and errors. This function will retry in the case of a // rate limit. -func Request(request rest.Request) (chan *rest.Response, chan error) { +func MakeRequestAsync(request rest.Request) (chan *rest.Response, chan error) { r := make(chan *rest.Response) e := make(chan error) go func() { - retry := 0 - for { - response, err := DefaultClient.API(request) - if err != nil { - e <- err - return - } - - if response.StatusCode == http.StatusTooManyRequests { - if retry > rateLimitRetry { - e <- errors.New("Rate limit retry exceeded") - return - } - retry++ - time.Sleep(rateLimitSleep * time.Millisecond) - continue - } - + response, err := MakeRequestRetry(request) + if err != nil { + e <- err + } + if response != nil { r <- response } }() From 46ae094fdc7a04c3edb2394e0bdc1b9c3e87037c Mon Sep 17 00:00:00 2001 From: Andy Trimble Date: Thu, 12 Oct 2017 14:18:59 -0600 Subject: [PATCH 3/6] Fixing for Go 1.6 and 1.7. --- sendgrid.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sendgrid.go b/sendgrid.go index e7baafd7..b4699b25 100644 --- a/sendgrid.go +++ b/sendgrid.go @@ -99,7 +99,7 @@ func MakeRequestRetry(request rest.Request) (*rest.Response, error) { resetTime = time.Unix(int64(t), 0) } } - time.Sleep(time.Until(resetTime)) + time.Sleep(resetTime.Sub(time.Now())) } } From 02a8ac226a7da72553fdc2b107d2b6bea5efe69f Mon Sep 17 00:00:00 2001 From: Andy Trimble Date: Thu, 12 Oct 2017 14:49:35 -0600 Subject: [PATCH 4/6] Added tests. --- sendgrid_test.go | 101 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/sendgrid_test.go b/sendgrid_test.go index c0457345..74c60e8e 100644 --- a/sendgrid_test.go +++ b/sendgrid_test.go @@ -11,6 +11,7 @@ import ( "os/exec" "path/filepath" "runtime" + "strconv" "strings" "testing" "time" @@ -167,6 +168,106 @@ func TestCustomHTTPClient(t *testing.T) { } } +func TestRequestRetry_rateLimit(t *testing.T) { + fakeServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-RateLimit-Reset", strconv.Itoa(int(time.Until(time.Now().Add(1*time.Second)).Seconds()))) + w.WriteHeader(http.StatusTooManyRequests) + })) + defer fakeServer.Close() + apiKey := "SENDGRID_APIKEY" + host := fakeServer.URL + request := GetRequest(apiKey, "/v3/test_endpoint", host) + request.Method = "GET" + var custom rest.Client + custom.HTTPClient = &http.Client{Timeout: time.Millisecond * 10} + DefaultClient = &custom + _, err := MakeRequestRetry(request) + if err == nil { + t.Error("An error did not trigger") + } + if !strings.Contains(err.Error(), "Rate limit retry exceeded") { + t.Error("We did not receive the rate limit error") + } + DefaultClient = rest.DefaultClient +} + +func TestRequestRetry_rateLimit_noHeader(t *testing.T) { + fakeServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusTooManyRequests) + })) + defer fakeServer.Close() + apiKey := "SENDGRID_APIKEY" + host := fakeServer.URL + request := GetRequest(apiKey, "/v3/test_endpoint", host) + request.Method = "GET" + var custom rest.Client + custom.HTTPClient = &http.Client{Timeout: time.Millisecond * 10} + DefaultClient = &custom + _, err := MakeRequestRetry(request) + if err == nil { + t.Error("An error did not trigger") + } + if !strings.Contains(err.Error(), "Rate limit retry exceeded") { + t.Error("We did not receive the rate limit error") + } + DefaultClient = rest.DefaultClient +} + +func TestRequestAsync(t *testing.T) { + fakeServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer fakeServer.Close() + apiKey := "SENDGRID_APIKEY" + host := fakeServer.URL + request := GetRequest(apiKey, "/v3/test_endpoint", host) + request.Method = "GET" + var custom rest.Client + custom.HTTPClient = &http.Client{Timeout: time.Millisecond * 10} + DefaultClient = &custom + r, e := MakeRequestAsync(request) + + select { + case <-r: + case err := <-e: + t.Errorf("Received an error,:%v", err) + case <-time.After(10 * time.Second): + t.Error("Timed out waiting for a response") + } + DefaultClient = rest.DefaultClient +} + +func TestRequestAsync_rateLimit(t *testing.T) { + fakeServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-RateLimit-Reset", strconv.Itoa(int(time.Until(time.Now().Add(1*time.Second)).Seconds()))) + w.WriteHeader(http.StatusTooManyRequests) + })) + defer fakeServer.Close() + apiKey := "SENDGRID_APIKEY" + host := fakeServer.URL + request := GetRequest(apiKey, "/v3/test_endpoint", host) + request.Method = "GET" + var custom rest.Client + custom.HTTPClient = &http.Client{Timeout: time.Millisecond * 10} + DefaultClient = &custom + r, e := MakeRequestAsync(request) + + select { + case <-r: + t.Error("Received a valid response") + return + case err := <-e: + if err == nil { + if !strings.Contains(err.Error(), "Rate limit retry exceeded") { + t.Error("We did not receive the rate limit error") + } + } + case <-time.After(10 * time.Second): + t.Error("Timed out waiting for an error") + } + DefaultClient = rest.DefaultClient +} + func Test_test_access_settings_activity_get(t *testing.T) { apiKey := "SENDGRID_APIKEY" host := "http://localhost:4010" From 7f8d72cdf7f78d17ea4adbcffc10071d8bd2e179 Mon Sep 17 00:00:00 2001 From: Andy Trimble Date: Thu, 12 Oct 2017 14:52:00 -0600 Subject: [PATCH 5/6] Fixed copy and paste bug in a test. --- sendgrid_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sendgrid_test.go b/sendgrid_test.go index 74c60e8e..2c0f68f2 100644 --- a/sendgrid_test.go +++ b/sendgrid_test.go @@ -257,10 +257,8 @@ func TestRequestAsync_rateLimit(t *testing.T) { t.Error("Received a valid response") return case err := <-e: - if err == nil { - if !strings.Contains(err.Error(), "Rate limit retry exceeded") { - t.Error("We did not receive the rate limit error") - } + if !strings.Contains(err.Error(), "Rate limit retry exceeded") { + t.Error("We did not receive the rate limit error") } case <-time.After(10 * time.Second): t.Error("Timed out waiting for an error") From 7ee828a6a4800680d01143b43e6f5d587dc3d3b7 Mon Sep 17 00:00:00 2001 From: Andy Trimble Date: Thu, 12 Oct 2017 15:02:26 -0600 Subject: [PATCH 6/6] Fixing test for Go 1.6 and 1.7. --- sendgrid_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sendgrid_test.go b/sendgrid_test.go index 2c0f68f2..9f95dd60 100644 --- a/sendgrid_test.go +++ b/sendgrid_test.go @@ -170,7 +170,7 @@ func TestCustomHTTPClient(t *testing.T) { func TestRequestRetry_rateLimit(t *testing.T) { fakeServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("X-RateLimit-Reset", strconv.Itoa(int(time.Until(time.Now().Add(1*time.Second)).Seconds()))) + w.Header().Set("X-RateLimit-Reset", strconv.Itoa(int(time.Now().Add(1*time.Second).Unix()))) w.WriteHeader(http.StatusTooManyRequests) })) defer fakeServer.Close() @@ -239,7 +239,7 @@ func TestRequestAsync(t *testing.T) { func TestRequestAsync_rateLimit(t *testing.T) { fakeServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("X-RateLimit-Reset", strconv.Itoa(int(time.Until(time.Now().Add(1*time.Second)).Seconds()))) + w.Header().Set("X-RateLimit-Reset", strconv.Itoa(int(time.Now().Add(1*time.Second).Unix()))) w.WriteHeader(http.StatusTooManyRequests) })) defer fakeServer.Close()