Generalize the VDP API

This commit is to prepare use of the VDP API also for the backend
side to filter bereq.body through bereq.filters by putting all pointers
_intended_ to be used by a VDP init function into vdp_ctx.

For background, see #4035

bin/varnishd/cache/cache_deliver_proc.c

@@ -71,18 +71,35 @@ VDP_Fini(const struct vdp_ctx *vdc)
 
 void
 VDP_Init(struct vdp_ctx *vdc, struct worker *wrk, struct vsl_log *vsl,
-    struct req *req)
+    struct req *req, struct busyobj *bo, intmax_t *clen)
 {
 	AN(vdc);
 	CHECK_OBJ_NOTNULL(wrk, WORKER_MAGIC);
 	AN(vsl);
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+
+	CHECK_OBJ_ORNULL(req, REQ_MAGIC);
+	CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC);
+	AN(clen);
+
+	assert((req ? 1 : 0) ^ (bo ? 1 : 0));
+
+	AN(clen);
+	assert(*clen >= -1);
 
 	INIT_OBJ(vdc, VDP_CTX_MAGIC);
 	VTAILQ_INIT(&vdc->vdp);
 	vdc->wrk = wrk;
 	vdc->vsl = vsl;
-	vdc->req = req;
+	vdc->clen = clen;
+
+	if (req != NULL) {
+		vdc->oc = req->objcore;
+		vdc->hp = req->resp;
+	}
+	else {
+		vdc->oc = bo->bereq_body;
+		vdc->hp = bo->bereq;
+	}
 }
 
 /* VDP_bytes
@@ -152,6 +169,10 @@ VDP_Push(VRT_CTX, struct vdp_ctx *vdc, struct ws *ws, const struct vdp *vdp,
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC);
+	CHECK_OBJ_ORNULL(vdc->oc, OBJCORE_MAGIC);
+	CHECK_OBJ_NOTNULL(vdc->hp, HTTP_MAGIC);
+	AN(vdc->clen);
+	assert(*vdc->clen >= -1);
 	AN(ws);
 	AN(vdp);
 	AN(vdp->name);
@@ -175,10 +196,9 @@ VDP_Push(VRT_CTX, struct vdp_ctx *vdc, struct ws *ws, const struct vdp *vdp,
 	vdc->nxt = VTAILQ_FIRST(&vdc->vdp);
 
 	AZ(vdc->retval);
-	if (vdpe->vdp->init != NULL) {
-		vdc->retval = vdpe->vdp->init(ctx, vdc, &vdpe->priv,
-		    vdpe == vdc->nxt ? vdc->req->objcore : NULL);
-	}
+	if (vdpe->vdp->init != NULL)
+		vdc->retval = vdpe->vdp->init(ctx, vdc, &vdpe->priv);
+	vdc->oc = NULL;
 
 	if (vdc->retval > 0) {
 		VTAILQ_REMOVE(&vdc->vdp, vdpe, list);
@@ -254,7 +274,9 @@ VDP_DeliverObj(struct vdp_ctx *vdc, struct objcore *oc)
 	CHECK_OBJ_NOTNULL(oc, OBJCORE_MAGIC);
 	CHECK_OBJ_NOTNULL(vdc->wrk, WORKER_MAGIC);
 	AN(vdc->vsl);
-	vdc->req = NULL;
+	AZ(vdc->oc);
+	vdc->hp = NULL;
+	vdc->clen = NULL;
 	final = oc->flags & (OC_F_PRIVATE | OC_F_HFM | OC_F_HFP) ? 1 : 0;
 	r = ObjIterate(vdc->wrk, oc, vdc, vdp_objiterate, final);
 	if (r < 0)

bin/varnishd/cache/cache_esi_deliver.c

@@ -261,35 +261,41 @@ ved_decode_len(struct vsl_log *vsl, const uint8_t **pp)
  */
 
 static int v_matchproto_(vdp_init_f)
-ved_vdp_esi_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc)
+ved_vdp_esi_init(VRT_CTX, struct vdp_ctx *vdc, void **priv)
 {
 	struct ecx *ecx;
-	struct req *req;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_ORNULL(ctx->req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC);
-	CHECK_OBJ_ORNULL(oc, OBJCORE_MAGIC);
-	if (oc == NULL || !ObjHasAttr(vdc->wrk, oc, OA_ESIDATA))
-		return (1);
-
-	req = vdc->req;
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_ORNULL(vdc->oc, OBJCORE_MAGIC);
+	CHECK_OBJ_NOTNULL(vdc->hp, HTTP_MAGIC);
+	AN(vdc->clen);
 	AN(priv);
+
 	AZ(*priv);
+	if (vdc->oc == NULL || !ObjHasAttr(vdc->wrk, vdc->oc, OA_ESIDATA))
+		return (1);
+
+	if (ctx->req == NULL) {
+		VSLb(vdc->vsl, SLT_Error,
+		     "esi can only be used on the client side");
+		return (1);
+	}
 
 	ALLOC_OBJ(ecx, ECX_MAGIC);
 	AN(ecx);
 	assert(sizeof gzip_hdr == 10);
-	ecx->preq = req;
+	ecx->preq = ctx->req;
 	*priv = ecx;
-	RFC2616_Weaken_Etag(req->resp);
-
-	req->res_mode |= RES_ESI;
-	if (req->resp_len != 0)
-		req->resp_len = -1;
-	if (req->esi_level > 0) {
-		assert(req->transport == &VED_transport);
-		CAST_OBJ_NOTNULL(ecx->pecx, req->transport_priv, ECX_MAGIC);
+	RFC2616_Weaken_Etag(vdc->hp);
+
+	ctx->req->res_mode |= RES_ESI;
+	if (*vdc->clen != 0)
+		*vdc->clen = -1;
+	if (ctx->req->esi_level > 0) {
+		assert(ctx->req->transport == &VED_transport);
+		CAST_OBJ_NOTNULL(ecx->pecx, ctx->req->transport_priv, ECX_MAGIC);
 		if (!ecx->pecx->isgzip)
 			ecx->pecx = NULL;
 	}
@@ -604,18 +610,16 @@ struct ved_foo {
 };
 
 static int v_matchproto_(vdp_init_f)
-ved_gzgz_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc)
+ved_gzgz_init(VRT_CTX, struct vdp_ctx *vdc, void **priv)
 {
 	ssize_t l;
 	const char *p;
 	struct ved_foo *foo;
-	struct req *req;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC);
-	(void)oc;
-	req = vdc->req;
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	AN(priv);
+
 	CAST_OBJ_NOTNULL(foo, *priv, VED_FOO_MAGIC);
 	CHECK_OBJ_NOTNULL(foo->objcore, OBJCORE_MAGIC);
 
@@ -925,7 +929,6 @@ ved_deliver(struct req *req, struct boc *boc, int wantbody)
 		foo->ecx = ecx;
 		foo->objcore = req->objcore;
 		i = VDP_Push(ctx, req->vdc, req->ws, &ved_gzgz, foo);
-
 	} else if (ecx->isgzip && !i) {
 		/* Non-Gzip'ed include in gzip'ed parent */
 		i = VDP_Push(ctx, req->vdc, req->ws, &ved_pretend_gz, ecx);

bin/varnishd/cache/cache_filter.h

@@ -90,7 +90,7 @@ struct vfp_ctx {
 	struct http		*req;
 	struct http		*resp;
 	struct worker		*wrk;
-	struct objcore		*oc;
+	struct objcore		*oc; // Only first filter, if at all
 
 	struct vfp_entry_s	vfp;
 	struct vfp_entry	*vfp_nxt;
@@ -112,8 +112,8 @@ enum vdp_action {
 	VDP_END,		/* Last buffer or after, implies VDP_FLUSH */
 };
 
-typedef int vdp_init_f(VRT_CTX, struct vdp_ctx *, void **priv,
-    struct objcore *);
+
+typedef int vdp_init_f(VRT_CTX, struct vdp_ctx *, void **priv);
 /*
  * Return value:
  *	negative:	Error - abandon delivery
@@ -155,7 +155,11 @@ struct vdp_ctx {
 	struct vdp_entry	*nxt;
 	struct worker		*wrk;
 	struct vsl_log		*vsl;
-	struct req		*req;
+	// NULL'ed after the first filter has been pushed
+	struct objcore		*oc;
+	// NULL'ed for delivery
+	struct http		*hp;
+	intmax_t		*clen;
 };
 
 int VDP_bytes(struct vdp_ctx *, enum vdp_action act, const void *, ssize_t);

bin/varnishd/cache/cache_gzip.c

@@ -288,21 +288,21 @@ VGZ_Gzip(struct vgz *vg, const void **pptr, ssize_t *plen, enum vgz_flag flags)
  */
 
 static int v_matchproto_(vdp_init_f)
-vdp_gunzip_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc)
+vdp_gunzip_init(VRT_CTX, struct vdp_ctx *vdc, void **priv)
 {
 	struct vgz *vg;
 	struct boc *boc;
-	struct req *req;
 	enum boc_state_e bos;
 	const char *p;
 	ssize_t dl;
 	uint64_t u;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
 	CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC);
-	CHECK_OBJ_ORNULL(oc, OBJCORE_MAGIC);
-	req = vdc->req;
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
+	CHECK_OBJ_ORNULL(vdc->oc, OBJCORE_MAGIC);
+	CHECK_OBJ_NOTNULL(vdc->hp, HTTP_MAGIC);
+	AN(vdc->clen);
+	AN(priv);
 
 	vg = VGZ_NewGunzip(vdc->vsl, "U D -");
 	AN(vg);
@@ -314,27 +314,27 @@ vdp_gunzip_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc)
 	VGZ_Obuf(vg, vg->m_buf, vg->m_sz);
 	*priv = vg;
 
-	http_Unset(req->resp, H_Content_Encoding);
+	http_Unset(vdc->hp, H_Content_Encoding);
 
-	req->resp_len = -1;
+	*vdc->clen = -1;
 
-	if (oc == NULL)
+	if (vdc->oc == NULL)
 		return (0);
 
-	boc = HSH_RefBoc(oc);
+	boc = HSH_RefBoc(vdc->oc);
 	if (boc != NULL) {
 		CHECK_OBJ(boc, BOC_MAGIC);
 		bos = boc->state;
-		HSH_DerefBoc(vdc->wrk, oc);
+		HSH_DerefBoc(vdc->wrk, vdc->oc);
 		if (bos < BOS_FINISHED)
 			return (0); /* OA_GZIPBITS is not stable yet */
 	}
 
-	p = ObjGetAttr(vdc->wrk, oc, OA_GZIPBITS, &dl);
+	p = ObjGetAttr(vdc->wrk, vdc->oc, OA_GZIPBITS, &dl);
 	if (p != NULL && dl == 32) {
 		u = vbe64dec(p + 24);
 		if (u != 0)
-			req->resp_len = u;
+			*vdc->clen = u;
 	}
 	return (0);
 }

bin/varnishd/cache/cache_range.c

@@ -196,33 +196,40 @@ vrg_ifrange(struct req *req)
 }
 
 static int v_matchproto_(vdp_init_f)
-vrg_range_init(VRT_CTX, struct vdp_ctx *vdc, void **priv, struct objcore *oc)
+vrg_range_init(VRT_CTX, struct vdp_ctx *vdc, void **priv)
 {
 	const char *err;
-	struct req *req;
 
 	CHECK_OBJ_NOTNULL(ctx, VRT_CTX_MAGIC);
+	CHECK_OBJ_ORNULL(ctx->req, REQ_MAGIC);
 	CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC);
-	(void)oc;
-	req = vdc->req;
-	CHECK_OBJ_NOTNULL(req, REQ_MAGIC);
-	if (!vrg_ifrange(req))		// rfc7233,l,455,456
+	AN(priv);
+
+	if (ctx->req == NULL) {
+		VSLb(vdc->vsl, SLT_Error,
+		     "range can only be used on the client side");
+		return (1);
+	}
+
+	// not using vdc->{hd,cl}, because range needs req anyway for Req_Fail()
+
+	if (!vrg_ifrange(ctx->req))		// rfc7233,l,455,456
 		return (1);
-	err = vrg_dorange(req, priv);
+	err = vrg_dorange(ctx->req, priv);
 	if (err == NULL)
 		return (*priv == NULL ? 1 : 0);
 
 	VSLb(vdc->vsl, SLT_Debug, "RANGE_FAIL %s", err);
-	if (req->resp_len >= 0)
-		http_PrintfHeader(req->resp,
+	if (ctx->req->resp_len >= 0)
+		http_PrintfHeader(ctx->req->resp,
 		    "Content-Range: bytes */%jd",
-		    (intmax_t)req->resp_len);
-	http_PutResponse(req->resp, "HTTP/1.1", 416, NULL);
+		    (intmax_t)ctx->req->resp_len);
+	http_PutResponse(ctx->req->resp, "HTTP/1.1", 416, NULL);
 	/*
 	 * XXX: We ought to produce a body explaining things.
 	 * XXX: That really calls for us to hit vcl_synth{}
 	 */
-	req->resp_len = 0;
+	ctx->req->resp_len = 0;
 	return (1);
 }
 

bin/varnishd/cache/cache_req_fsm.c

@@ -462,11 +462,11 @@ cnt_transmit(struct worker *wrk, struct req *req)
 		sendbody = 1;
 	}
 
-	VDP_Init(req->vdc, req->wrk, req->vsl, req);
+	VDP_Init(req->vdc, req->wrk, req->vsl, req, NULL, &req->resp_len);
 	if (req->vdp_filter_list == NULL)
 		req->vdp_filter_list = resp_Get_Filter_List(req);
 	if (req->vdp_filter_list == NULL ||
-	    VCL_StackVDP(req, req->vcl, req->vdp_filter_list)) {
+	    VCL_StackVDP(req->vdc, req->vcl, req->vdp_filter_list, req, NULL)) {
 		VSLb(req->vsl, SLT_Error, "Failure to push processors");
 		req->doclose = SC_OVERLOAD;
 		req->acct.resp_bodybytes +=

bin/varnishd/cache/cache_varnishd.h

@@ -193,7 +193,7 @@ void VDI_Init(void);
 /* cache_deliver_proc.c */
 void VDP_Fini(const struct vdp_ctx *vdc);
 void VDP_Init(struct vdp_ctx *vdc, struct worker *wrk, struct vsl_log *vsl,
-    struct req *req);
+    struct req *req, struct busyobj *bo, intmax_t *cl);
 uint64_t VDP_Close(struct vdp_ctx *, struct objcore *, struct boc *);
 void VDP_Panic(struct vsb *vsb, const struct vdp_ctx *vdc);
 int VDP_Push(VRT_CTX, struct vdp_ctx *, struct ws *, const struct vdp *,
@@ -511,7 +511,8 @@ void pan_privs(struct vsb *, const struct vrt_privs *);
 
 /* cache_vrt_filter.c */
 int VCL_StackVFP(struct vfp_ctx *, const struct vcl *, const char *);
-int VCL_StackVDP(struct req *, const struct vcl *, const char *);
+int VCL_StackVDP(struct vdp_ctx *vdc, const struct vcl *vcl, const char *fl,
+    struct req *req, struct busyobj *bo);
 const char *resp_Get_Filter_List(struct req *req);
 void VCL_VRT_Init(void);
 

bin/varnishd/cache/cache_vrt_filter.c

@@ -254,15 +254,28 @@ VCL_StackVFP(struct vfp_ctx *vc, const struct vcl *vcl, const char *fl)
 }
 
 int
-VCL_StackVDP(struct req *req, const struct vcl *vcl, const char *fl)
+VCL_StackVDP(struct vdp_ctx *vdc, const struct vcl *vcl, const char *fl,
+    struct req *req, struct busyobj *bo)
 {
 	const struct vfilter *vp;
 	struct vrt_ctx ctx[1];
 
+	CHECK_OBJ_NOTNULL(vdc, VDP_CTX_MAGIC);
+	AN(vcl);
 	AN(fl);
-	VSLbs(req->vsl, SLT_Filters, TOSTRAND(fl));
+
+	CHECK_OBJ_ORNULL(req, REQ_MAGIC);
+	CHECK_OBJ_ORNULL(bo, BUSYOBJ_MAGIC);
+
+	assert((req ? 1 : 0) ^ (bo ? 1 : 0));
+
+	VSLbs(vdc->vsl, SLT_Filters, TOSTRAND(fl));
 	INIT_OBJ(ctx, VRT_CTX_MAGIC);
-	VCL_Req2Ctx(ctx, req);
+
+	if (req)
+		VCL_Req2Ctx(ctx, req);
+	else
+		VCL_Bo2Ctx(ctx, bo);
 
 	while (1) {
 		vp = vcl_filter_list_iter(0, &vrt_filters, &vcl->filters, &fl);
@@ -273,7 +286,7 @@ VCL_StackVDP(struct req *req, const struct vcl *vcl, const char *fl)
 			    "Filter '...%s' not found", fl);
 			return (-1);
 		}
-		if (VDP_Push(ctx, req->vdc, req->ws, vp->vdp, NULL))
+		if (VDP_Push(ctx, vdc, ctx->ws, vp->vdp, NULL))
 			return (-1);
 	}
 }