refactor ø.Bytes to accept io.Writer

doc/user-guide.md

@@ -9,22 +9,22 @@ This is a "combinator" library for network I/O. Combinators open up an opportuni
 
 - [Background](#background)
   - [Combinators](#combinators)
-  - [High-Order Functions](#high-order-functions)
+  - [High-order functions](#high-order-functions)
   - [Lifecycle](#lifecycle)
 - [Import library](#import-library)
 - [Writer combinators](#writer-combinators)
   - [Method](#method)
   - [Target URI](#target-uri)
   - [Query Params](#query-params)
   - [Request Headers](#request-headers)
-  - [Request Payload](#request-payload)
+  - [Request payload](#request-payload)
 - [Reader combinators](#reader-combinators)
   - [Status Code](#status-code)
   - [Response Headers](#response-headers)
   - [Response Payload](#response-payload)
   - [Assert Payload](#assert-payload)
   - [Using Variables for Dynamic Behavior](#using-variables-for-dynamic-behavior)
-- [Chain Networking I/O](#chain-networking-io)
+- [Chain networking I/O](#chain-networking-io)
 
 ---
 
@@ -397,11 +397,11 @@ For all other cases, there is `ƒ.Bytes` combinator that receives raw binaries.
 
 ```go
 func SomeXxx() http.Arrow {
-  var data []byte
+  data := &bytes.Buffer{}
 
   return http.GET(
     // ...
-    ƒ.Bytes(&data), // Note: pointer to buffer is required
+    ƒ.Bytes(data), // Note: pointer to buffer is required
   )
 }
 ```

go.mod

@@ -1,6 +1,6 @@
 module github.com/fogfish/gurl/v2
 
-go 1.20
+go 1.21
 
 require (
 	github.com/ajg/form v1.5.2-0.20200323032839-9aeb3cf462e1

http/mock/mock.go

@@ -0,0 +1,97 @@
+//
+// Copyright (C) 2019 - 2024 Dmitry Kolesnikov
+//
+// This file may be modified and distributed under the terms
+// of the MIT license.  See the LICENSE file for details.
+// https://github.com/fogfish/gurl
+//
+
+package mock
+
+import (
+	"bytes"
+	µ "github.com/fogfish/gurl/v2/http"
+	"io"
+	"net/http"
+)
+
+// Mocks HTTP client
+type Mock struct {
+	r   *http.Response
+	err error
+}
+
+func (mock *Mock) Do(req *http.Request) (*http.Response, error) {
+	if mock.err != nil {
+		return nil, mock.err
+	}
+
+	return mock.r, nil
+}
+
+// Mocks Option
+type Option func(m *Mock)
+
+func Preset(opts ...Option) Option {
+	return func(m *Mock) {
+		for _, opt := range opts {
+			opt(m)
+		}
+	}
+}
+
+// Mock failure of HTTP client
+func Fail(err error) Option {
+	return func(m *Mock) {
+		m.err = err
+	}
+}
+
+// Mock response with status code (default 200)
+func Status(code int) Option {
+	return func(m *Mock) {
+		m.r.StatusCode = code
+	}
+}
+
+// Mock response with HTTP header (default none)
+func Header(h, v string) Option {
+	return func(m *Mock) {
+		m.r.Header.Set(h, v)
+	}
+}
+
+// Mock response body (default empty)
+func Body(body []byte) Option {
+	return func(m *Mock) {
+		m.r.Body = io.NopCloser(bytes.NewBuffer(body))
+	}
+}
+
+// Mock response body I/O error (default nil)
+func IOError(err error) Option {
+	return func(m *Mock) {
+		m.r.Body = io.NopCloser(errReader{err})
+	}
+}
+
+// Mock HTTP Client
+func New(opts ...Option) µ.Config {
+	m := &Mock{
+		r: &http.Response{
+			StatusCode: http.StatusOK,
+			Header:     http.Header{},
+			Body:       io.NopCloser(bytes.NewBuffer([]byte{})),
+		},
+	}
+
+	for _, opt := range opts {
+		opt(m)
+	}
+
+	return µ.WithClient(m)
+}
+
+type errReader struct{ err error }
+
+func (r errReader) Read(p []byte) (n int, err error) { return 0, r.err }

http/recv/arrows.go

@@ -677,9 +677,29 @@ func Expect[T any](expect T) http.Arrow {
 }
 
 // Bytes receive raw binary from HTTP response
-func Bytes(val *[]byte) http.Arrow {
+func Bytes(w io.Writer) http.Arrow {
 	return func(cat *http.Context) (err error) {
-		*val, err = io.ReadAll(cat.Response.Body)
+		var n int
+		buf := make([]byte, 64*1024) // 64KB is size of chunk to be processed once
+		for {
+			n, err = cat.Response.Body.Read(buf)
+			if err == io.EOF {
+				err = nil
+				// There may be one last chunk to receive before breaking the loop.
+				if n <= 0 {
+					break
+				}
+			}
+			if err != nil {
+				break
+			}
+
+			_, err = w.Write(buf[:n])
+			if err != nil {
+				break
+			}
+		}
+
 		cat.Response.Body.Close()
 		cat.Response = nil
 		return

http/recv/arrows_test.go

@@ -9,8 +9,10 @@
 package recv_test
 
 import (
+	"bytes"
 	"context"
 	"encoding/base64"
+	"errors"
 	"image"
 	_ "image/png"
 	"net/http"
@@ -21,6 +23,7 @@ import (
 	"time"
 
 	µ "github.com/fogfish/gurl/v2/http"
+	iomock "github.com/fogfish/gurl/v2/http/mock"
 	ƒ "github.com/fogfish/gurl/v2/http/recv"
 	ø "github.com/fogfish/gurl/v2/http/send"
 	"github.com/fogfish/it/v2"
@@ -379,33 +382,61 @@ func TestExpectJSONFailed(t *testing.T) {
 }
 
 func TestRecvBytes(t *testing.T) {
-	ts := mock()
-	defer ts.Close()
+	opts := iomock.Preset(
+		iomock.Status(http.StatusOK),
+		iomock.Body([]byte("site=example.com")),
+	)
 
-	for path, content := range map[string]µ.Arrow{
-		"/text":   ƒ.ContentType.Text,
-		"/text/1": ƒ.ContentType.TextPlain,
-		"/html":   ƒ.ContentType.HTML,
-		"/html/2": ƒ.ContentType.TextHTML,
+	for _, content := range []struct {
+		arrow  µ.Arrow
+		header string
+	}{
+		{ƒ.ContentType.Text, "text/plain"},
+		{ƒ.ContentType.TextPlain, "text/plain"},
+		{ƒ.ContentType.HTML, "text/html"},
+		{ƒ.ContentType.TextHTML, "text/html"},
 	} {
 
-		var data []byte
+		data := &bytes.Buffer{}
 		req := µ.GET(
-			ø.URI(ts.URL+path),
+			ø.URI("http://example.com/test"),
 			ƒ.Status.OK,
-			content,
-			ƒ.Bytes(&data),
+			content.arrow,
+			ƒ.Bytes(data),
 		)
-		cat := µ.New()
+		cat := µ.New(iomock.New(opts, iomock.Header("Content-Type", content.header)))
 		err := cat.IO(context.Background(), req)
 
 		it.Then(t).Should(
 			it.Nil(err),
-			it.Equal(string(data), "site=example.com"),
+			it.Equal(data.String(), "site=example.com"),
 		)
 	}
 }
 
+func TestRecvBytesFail(t *testing.T) {
+	opts := iomock.Preset(
+		iomock.Status(http.StatusOK),
+		iomock.Header("Content-Type", "text/plain"),
+		iomock.IOError(errors.New("i/o error")),
+	)
+
+	data := &bytes.Buffer{}
+	req := µ.GET(
+		ø.URI("http://example.com/test"),
+		ƒ.Status.OK,
+		ƒ.ContentType.Text,
+		ƒ.Bytes(data),
+	)
+	cat := µ.New(iomock.New(opts))
+	err := cat.IO(context.Background(), req)
+
+	it.Then(t).ShouldNot(
+		it.Nil(err),
+		it.Equal(data.String(), "site=example.com"),
+	)
+}
+
 func TestMatch(t *testing.T) {
 	ts := mock()
 	defer ts.Close()

awsapi/go.mod → http/x/awsapi/go.mod

@@ -1,4 +1,4 @@
-module github.com/fogfish/gurl/awsapi
+module github.com/fogfish/gurl/http/x/awsapi
 
 go 1.21.0