Merge branch 'master' into missing-arrayAttributes

CHANGELOG.md

@@ -14,10 +14,12 @@ Unreleased changes are available as `avenga/couper:edge` container.
 
 * **Fixed**
   * Handling of [`accept_forwarded_url`](./docs/REFERENCE.md#settings-block) "host" if `H-Forwarded-Host` request header field contains a port ([#360](https://github.com/avenga/couper/pull/360))
-  * Setting `Vary` response header fields for [CORS](./doc/REFERENCE.md#cors-block) ([#362](https://github.com/avenga/couper/pull/362))
+  * Setting `Vary` response header fields for [CORS](./docs/REFERENCE.md#cors-block) ([#362](https://github.com/avenga/couper/pull/362))
   * Use of referenced backends in [OAuth2 CC Blocks](./docs/REFERENCE.md#oauth2-cc-block) ([#321](https://github.com/avenga/couper/issues/321))
-  * [CORS](./doc/REFERENCE.md#cors-block) preflight requests are not blocked by access controls any more ([#366](https://github.com/avenga/couper/pull/366))
-  * For each SAML attribute listed in [`array_attributes`](./doc/REFERENCE.md#saml-block) at least an empty array is created in `request.context.<label>.attributes.<name>` ([#369](https://github.com/avenga/couper/pull/369))
+  * [CORS](./docs/REFERENCE.md#cors-block) preflight requests are not blocked by access controls anymore ([#366](https://github.com/avenga/couper/pull/366))
+  * Reduced memory usage for backend response bodies which just get piped to the client and are not required to be read by Couper due to a variable references ([#375](https://github.com/avenga/couper/pull/375))
+    * However, if a huge message body is passed and additionally referenced via e.g. `json_body`, Couper may require a lot of memory for storing the data structure.
+  * For each SAML attribute listed in [`array_attributes`](./docs/REFERENCE.md#saml-block) at least an empty array is created in `request.context.<label>.attributes.<name>` ([#369](https://github.com/avenga/couper/pull/369))
 
 ---
 

Makefile

@@ -1,5 +1,5 @@
 .PHONY: docker-telemetry build generate image
-.PHONY: test test-docker test-coverage test-coverage-show
+.PHONY: test test-docker coverage test-coverage test-coverage-show
 
 build:
 	go build -race -v -o couper main.go
@@ -20,6 +20,8 @@ test:
 test-docker:
 	docker run --rm -v $(CURDIR):/go/app -w /go/app  golang sh -c "go test -short -count 10 -v -timeout 300s -race ./..."
 
+coverage: test-coverage test-coverage-show
+
 test-coverage:
 	go test -short -timeout 300s -covermode=count -coverprofile=ac.coverage ./accesscontrol
 	go test -short -timeout 300s -covermode=count -coverprofile=cache.coverage ./cache
@@ -32,7 +34,6 @@ test-coverage:
 	# go test -short -timeout 300s -covermode=count -coverprofile=logging.coverage ./logging
 	go test -short -timeout 300s -covermode=count -coverprofile=server.coverage ./server
 	go test -short -timeout 300s -covermode=count -coverprofile=main.coverage ./
-	$(MAKE) test-coverage-show
 
 test-coverage-show:
 	go tool cover -html=ac.coverage

config/request/context_key.go

@@ -7,6 +7,7 @@ const (
 	AccessControls
 	APIName
 	BackendName
+	BufferOptions
 	Endpoint
 	EndpointKind
 	Error

config/runtime/endpoint.go

@@ -76,6 +76,9 @@ func newEndpointOptions(confCtx *hcl.EvalContext, endpointConf *config.Endpoint,
 		errTpl = serverOptions.ServerErrTpl
 	}
 
+	// blockBodies contains inner endpoint block remain bodies to determine req/res buffer options.
+	var blockBodies []hcl.Body
+
 	var response *producer.Response
 	// var redirect producer.Redirect // TODO: configure redirect block
 	proxies := make(producer.Proxies, 0)
@@ -85,10 +88,11 @@ func newEndpointOptions(confCtx *hcl.EvalContext, endpointConf *config.Endpoint,
 		response = &producer.Response{
 			Context: endpointConf.Response.Remain,
 		}
+		blockBodies = append(blockBodies, response.Context)
 	}
 
 	for _, proxyConf := range endpointConf.Proxies {
-		backend, berr := newBackend(confCtx, proxyConf.Backend, log, proxyEnv, memStore)
+		backend, innerBody, berr := newBackend(confCtx, proxyConf.Backend, log, proxyEnv, memStore)
 		if berr != nil {
 			return nil, berr
 		}
@@ -98,10 +102,11 @@ func newEndpointOptions(confCtx *hcl.EvalContext, endpointConf *config.Endpoint,
 			RoundTrip: proxyHandler,
 		}
 		proxies = append(proxies, p)
+		blockBodies = append(blockBodies, proxyConf.Backend, innerBody, proxyConf.HCLBody())
 	}
 
 	for _, requestConf := range endpointConf.Requests {
-		backend, berr := newBackend(confCtx, requestConf.Backend, log, proxyEnv, memStore)
+		backend, innerBody, berr := newBackend(confCtx, requestConf.Backend, log, proxyEnv, memStore)
 		if berr != nil {
 			return nil, berr
 		}
@@ -111,6 +116,7 @@ func newEndpointOptions(confCtx *hcl.EvalContext, endpointConf *config.Endpoint,
 			Context: requestConf.Remain,
 			Name:    requestConf.Name,
 		})
+		blockBodies = append(blockBodies, requestConf.Backend, innerBody, requestConf.HCLBody())
 	}
 
 	backendConf := *DefaultBackendConf
@@ -138,8 +144,7 @@ func newEndpointOptions(confCtx *hcl.EvalContext, endpointConf *config.Endpoint,
 		}}
 	}
 
-	// TODO: determine request/backend_responses.*.body access in this context (all including backend) or for now:
-	bufferOpts := eval.MustBuffer(endpointConf.Remain)
+	bufferOpts := eval.MustBuffer(append(blockBodies, endpointConf.Remain)...)
 	if len(proxies)+len(requests) > 1 { // also buffer with more possible results
 		bufferOpts |= eval.BufferResponse
 	}
@@ -150,15 +155,15 @@ func newEndpointOptions(confCtx *hcl.EvalContext, endpointConf *config.Endpoint,
 	}
 
 	return &handler.EndpointOptions{
-		APIName:       apiName,
-		Context:       endpointConf.Remain,
-		Error:         errTpl,
-		LogPattern:    endpointConf.Pattern,
-		Proxies:       proxies,
-		ReqBodyLimit:  bodyLimit,
-		ReqBufferOpts: bufferOpts,
-		Requests:      requests,
-		Response:      response,
-		ServerOpts:    serverOptions,
+		APIName:      apiName,
+		Context:      endpointConf.Remain,
+		Error:        errTpl,
+		LogPattern:   endpointConf.Pattern,
+		Proxies:      proxies,
+		ReqBodyLimit: bodyLimit,
+		BufferOpts:   bufferOpts,
+		Requests:     requests,
+		Response:     response,
+		ServerOpts:   serverOptions,
 	}, nil
 }

config/runtime/server.go

@@ -313,16 +313,16 @@ func NewServerConfiguration(conf *config.Couper, log *logrus.Entry, memStore *ca
 }
 
 func newBackend(evalCtx *hcl.EvalContext, backendCtx hcl.Body, log *logrus.Entry,
-	ignoreProxyEnv bool, memStore *cache.MemoryStore) (http.RoundTripper, error) {
+	ignoreProxyEnv bool, memStore *cache.MemoryStore) (http.RoundTripper, hcl.Body, error) {
 	beConf := *DefaultBackendConf
 	if diags := gohcl.DecodeBody(backendCtx, evalCtx, &beConf); diags.HasErrors() {
-		return nil, diags
+		return nil, nil, diags
 	}
 
 	if beConf.Name == "" {
 		name, err := getBackendName(evalCtx, backendCtx)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		beConf.Name = name
 	}
@@ -337,20 +337,20 @@ func newBackend(evalCtx *hcl.EvalContext, backendCtx hcl.Body, log *logrus.Entry
 	}
 
 	if err := parseDuration(beConf.ConnectTimeout, &tc.ConnectTimeout); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	if err := parseDuration(beConf.TTFBTimeout, &tc.TTFBTimeout); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	if err := parseDuration(beConf.Timeout, &tc.Timeout); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	openAPIopts, err := validation.NewOpenAPIOptions(beConf.OpenAPI)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	options := &transport.BackendOptions{
@@ -360,41 +360,49 @@ func newBackend(evalCtx *hcl.EvalContext, backendCtx hcl.Body, log *logrus.Entry
 
 	oauthContent, _, _ := backendCtx.PartialContent(config.OAuthBlockSchema)
 	if oauthContent == nil {
-		return backend, nil
+		return backend, backendCtx, nil
 	}
 
 	if blocks := oauthContent.Blocks.OfType("oauth2"); len(blocks) > 0 {
-		beConf.OAuth2 = &config.OAuth2ReqAuth{}
+		return newAuthBackend(evalCtx, beConf, blocks, log, ignoreProxyEnv, memStore, backend)
+	}
 
-		if diags := gohcl.DecodeBody(blocks[0].Body, evalCtx, beConf.OAuth2); diags.HasErrors() {
-			return nil, diags
-		}
+	return backend, backendCtx, nil
+}
 
-		innerContent, _, diags := beConf.OAuth2.Remain.PartialContent(beConf.OAuth2.Schema(true))
-		if diags.HasErrors() {
-			return nil, diags
-		}
-		innerBackend := innerContent.Blocks.OfType("backend")[0] // backend block is set by configload
-		authBackend, authErr := newBackend(evalCtx, innerBackend.Body, log, ignoreProxyEnv, memStore)
-		if authErr != nil {
-			return nil, authErr
-		}
+func newAuthBackend(evalCtx *hcl.EvalContext, beConf config.Backend, blocks hcl.Blocks, log *logrus.Entry,
+	ignoreProxyEnv bool, memStore *cache.MemoryStore, backend http.RoundTripper) (http.RoundTripper, hcl.Body, error) {
 
-		// Set default value
-		if beConf.OAuth2.Retries == nil {
-			var one uint8 = 1
-			beConf.OAuth2.Retries = &one
-		}
+	beConf.OAuth2 = &config.OAuth2ReqAuth{}
 
-		oauth2Client, err := oauth2.NewOAuth2CC(beConf.OAuth2, authBackend)
-		if err != nil {
-			return nil, err
-		}
+	if diags := gohcl.DecodeBody(blocks[0].Body, evalCtx, beConf.OAuth2); diags.HasErrors() {
+		return nil, nil, diags
+	}
+
+	innerContent, _, diags := beConf.OAuth2.Remain.PartialContent(beConf.OAuth2.Schema(true))
+	if diags.HasErrors() {
+		return nil, nil, diags
+	}
 
-		return transport.NewOAuth2ReqAuth(beConf.OAuth2, memStore, oauth2Client, backend)
+	innerBackend := innerContent.Blocks.OfType("backend")[0] // backend block is set by configload
+	authBackend, body, authErr := newBackend(evalCtx, innerBackend.Body, log, ignoreProxyEnv, memStore)
+	if authErr != nil {
+		return nil, nil, authErr
+	}
+
+	// Set default value
+	if beConf.OAuth2.Retries == nil {
+		var one uint8 = 1
+		beConf.OAuth2.Retries = &one
+	}
+
+	oauth2Client, err := oauth2.NewOAuth2CC(beConf.OAuth2, authBackend)
+	if err != nil {
+		return nil, body, err
 	}
 
-	return backend, nil
+	rt, err := transport.NewOAuth2ReqAuth(beConf.OAuth2, memStore, oauth2Client, backend)
+	return rt, body, err
 }
 
 func getBackendName(evalCtx *hcl.EvalContext, backendCtx hcl.Body) (string, error) {
@@ -438,7 +446,7 @@ func configureOidcConfigs(conf *config.Couper, confCtx *hcl.EvalContext, log *lo
 	if conf.Definitions != nil {
 		for _, oidcConf := range conf.Definitions.OIDC {
 			confErr := errors.Configuration.Label(oidcConf.Name)
-			backend, err := newBackend(confCtx, oidcConf.Backend, log, conf.Settings.NoProxyFromEnv, memStore)
+			backend, _, err := newBackend(confCtx, oidcConf.Backend, log, conf.Settings.NoProxyFromEnv, memStore)
 			if err != nil {
 				return nil, confErr.With(err)
 			}
@@ -544,7 +552,7 @@ func configureAccessControls(conf *config.Couper, confCtx *hcl.EvalContext, log
 
 		for _, oauth2Conf := range conf.Definitions.OAuth2AC {
 			confErr := errors.Configuration.Label(oauth2Conf.Name)
-			backend, err := newBackend(confCtx, oauth2Conf.Backend, log, conf.Settings.NoProxyFromEnv, memStore)
+			backend, _, err := newBackend(confCtx, oauth2Conf.Backend, log, conf.Settings.NoProxyFromEnv, memStore)
 			if err != nil {
 				return nil, confErr.With(err)
 			}
@@ -598,7 +606,7 @@ func configureJWKS(jwtConf *config.JWT, conf *config.Couper, confContext *hcl.Ev
 	var backend http.RoundTripper
 
 	if jwtConf.Backend != nil {
-		b, err := newBackend(confContext, jwtConf.Backend, log, ignoreProxyEnv, memStore)
+		b, _, err := newBackend(confContext, jwtConf.Backend, log, ignoreProxyEnv, memStore)
 		if err != nil {
 			return nil, err
 		}