Skip hob by hop headers in trailers for responses.

Also remove all hop by hop headers from 'Trailer'
header.

fw/cache.c

@@ -72,6 +72,7 @@ static const TfwStr tfw_cache_raw_headers_304[] = {
 /* Flags stored in a Cache Entry. */
 #define TFW_CE_MUST_REVAL	0x0001		/* MUST revalidate if stale. */
 #define TFW_CE_STALE_IF_ERROR	0x0002
+#define TFW_CE_CHUNKED_BODY	0x0004		/* CE entry contains chunked body. */
 
 /*
  * @trec	- Database record descriptor;
@@ -2244,7 +2245,8 @@ tfw_cache_copy_resp(TDB *db, TfwCacheEntry *ce, TfwHttpResp *resp, TfwStr *rph,
 		{
 			int hid = field - resp->h_tbl->tbl;
 
-			if (!(field->flags & TFW_STR_TRAILER))
+			if ((!(field->flags & TFW_STR_TRAILER)) ||
+			    (field->flags & (TFW_STR_HBH_HDR | TFW_STR_NOCCPY_HDR)))
 				continue;
 
 			n = tfw_cache_h2_copy_hdr(db, ce, resp, hid, &p, &trec,
@@ -2258,12 +2260,14 @@ tfw_cache_copy_resp(TDB *db, TfwCacheEntry *ce, TfwHttpResp *resp, TfwStr *rph,
 
 	/* Write HTTP response body. */
 	ce->body = TDB_OFF(db->hdr, p);
-	if (test_bit(TFW_HTTP_B_CHUNKED, resp->flags))
+	if (test_bit(TFW_HTTP_B_CHUNKED, resp->flags)) {
+		ce->flags |= TFW_CE_CHUNKED_BODY;
 		r = tfw_cache_h2_copy_chunked_body(db, &ce->body_len, &p, &trec,
 						   resp, &resp->cut, &tot_len);
-	else
+	} else {
 		r = tfw_cache_h2_copy_body(db, &ce->body_len, &p, &trec,
 					   resp, &tot_len);
+	}
 
 	if (unlikely(r)) {
 		T_ERR("Cache: cannot copy HTTP body\n");
@@ -3041,6 +3045,7 @@ tfw_cache_build_resp(TfwHttpReq *req, TfwCacheEntry *ce, long age)
 						    TFW_VHOST_HDRMOD_RESP);
 	bool h2_mode = TFW_MSG_H2(req);
 	bool first = false;
+	bool chunked_body = false;
 
 	/*
 	 * The allocated response won't be checked by any filters and
@@ -3160,10 +3165,10 @@ tfw_cache_build_resp(TfwHttpReq *req, TfwCacheEntry *ce, long age)
 	it->frag = skb_shinfo(it->skb)->nr_frags - 1;
 
 write_body:
+	chunked_body = (ce->flags & TFW_CE_CHUNKED_BODY) && !h2_mode;
 	/* Fill skb with body from cache for HTTP/2 or HTTP/1.1 response. */
 	BUG_ON(p != TDB_PTR(db->hdr, ce->body));
 	if (ce->body_len && req->method != TFW_HTTP_METH_HEAD) {
-		bool chunked_body = trailers_trec != NULL && !h2_mode;
 		if (tfw_cache_build_resp_body(db, trec, it, p, ce->body_len,
 					      h2_mode, chunked_body))
 			goto free;
@@ -3186,15 +3191,16 @@ tfw_cache_build_resp(TfwHttpReq *req, TfwCacheEntry *ce, long age)
 		}
 		if (h2_mode)
 			resp->req->stream->xmit.t_len = t_len;
-		/*
-		 * For http1 we should add finishing \r\n after last
-		 * trailer.
-		 */
-		if (!h2_mode &&
-		    tfw_http_msg_expand_data(it, skb_head, &g_crlf, NULL))
-			goto free;
 	}
 
+	/*
+	 * For http1 we should add finishing \r\n after last
+	 * trailer or chunked body.
+	 */
+	if (chunked_body && req->method != TFW_HTTP_METH_HEAD
+	    && tfw_http_msg_expand_data(it, skb_head, &g_crlf, NULL))
+		goto free;
+
 	return resp;
 free:
 	tfw_http_msg_free((TfwHttpMsg *)resp);

fw/http.c

@@ -3652,6 +3652,10 @@ tfw_h1_adjust_req(TfwHttpReq *req)
 	if (r < 0)
 		return r;
 
+	r = tfw_http_msg_adjust_trailer_hdr(hm);
+	if (r < 0)
+		return r;
+
 	r = tfw_http_set_hdr_upgrade(hm, false);
 	if (r < 0)
 		return r;
@@ -5008,6 +5012,9 @@ tfw_h2_hpack_encode_trailer_headers(TfwHttpResp *resp)
 				 || TFW_STR_DUP(tgt)))
 			return -EINVAL;
 
+		if (tgt->flags & TFW_STR_HBH_HDR)
+			continue;
+
 		T_DBG3("%s: hid=%hu, d_num=%hu, nchunks=%u\n",
 		       __func__, hid, d_num, ht->tbl[hid].nchunks);
 
@@ -6966,6 +6973,10 @@ tfw_http_resp_process(TfwConn *conn, TfwStream *stream, struct sk_buff *skb,
 	if (unlikely(r))
 		return r;
 
+	r = tfw_http_msg_adjust_trailer_hdr(hmresp);
+	if (r < 0)
+		return r;
+
 	/*
 	 * We need to know if connection will be upgraded after response
 	 * forwarding here before sibling processing because after upgrade

fw/http.h

@@ -272,6 +272,7 @@ typedef struct {
  * @cache_ctl		- cache control data for a message;
  * @version		- HTTP version (1.0 and 1.1 are only supported);
  * @keep_alive		- the value of timeout specified in Keep-Alive header;
+ * @trailer_hid		- id of 'Trailer' header;
  * @content_length	- the value of Content-Length header field;
  * @flags		- message related flags. The flags are tested
  *			  concurrently, but concurrent updates aren't
@@ -300,6 +301,7 @@ typedef struct {
 	TfwCacheControl	cache_ctl;					\
 	unsigned char	version;					\
 	unsigned int	keep_alive;					\
+	unsigned int	trailer_hid;					\
 	unsigned long	content_length;					\
 	DECLARE_BITMAP	(flags, _TFW_HTTP_FLAGS_NUM);			\
 	TfwConn		*conn;						\

fw/http2.c

@@ -539,8 +539,20 @@ tfw_h2_stream_xmit_prepare_resp(TfwStream *stream)
 		stream->xmit.t_len = resp->mit.acc_len - acc;
 
 		if (unlikely(r))
-			T_WARN("Failed to encode trailers");
+			goto finish;
+		/*
+		 * If we do not add any trailers in response (all trailers
+		 * are hop by hop headers), we should remove reserved
+		 * FRAME_HEADER_SIZE.
+		 */
+		if (unlikely(!stream->xmit.t_len)) {
+			r = ss_skb_list_chop_head_tail(&resp->msg.skb_head,
+						       0, FRAME_HEADER_SIZE);
+		}
+
 
+		if (unlikely(r))
+			T_WARN("Failed to encode trailers");
 	}
 
 finish:

fw/http_msg.c

@@ -621,13 +621,18 @@ tfw_http_msg_hdr_close(TfwHttpMsg *hm)
 	parser->hdr.flags |= TFW_STR_COMPLETE;
 
 	/* Cumulate the trailer headers length */
-	if (is_srv_conn && parser->hdr.flags & TFW_STR_TRAILER) {
+	if (is_srv_conn) {
 		TfwHttpResp* resp = (TfwHttpResp*) hm;
 
-		resp->trailers_len += parser->hdr.len +
-			tfw_str_eolen(&parser->hdr);
+		if (unlikely(parser->hdr.flags & TFW_STR_TRAILER)) {
+			resp->trailers_len += parser->hdr.len +
+				tfw_str_eolen(&parser->hdr);
+		}
 	}
 
+	if (unlikely(parser->hdr.flags & TFW_STR_TRAILER_HDR))
+		hm->trailer_hid = ht->off;
+
 	/*
 	 * We make this frang check here, because it is the earliest
 	 * place where we can determine that new added header is violating
@@ -898,6 +903,35 @@ __hdr_del(TfwHttpMsg *hm, unsigned int hid)
 	return 0;
 }
 
+static int
+__hdr_del_part(TfwHttpMsg *hm, unsigned int hid, unsigned flags)
+{
+	TfwHttpHdrTbl *ht = hm->h_tbl;
+	TfwStr *dup, *end, *hdr = &ht->tbl[hid];
+
+	TFW_STR_FOR_EACH_DUP(dup, hdr, end) {
+		TfwStr *c, *c_end;
+		int r = 0;
+		int id = 0;
+
+		TFW_STR_FOR_EACH_CHUNK(c, dup, c_end) {
+			if (!(c->flags & flags)) {
+				id++;
+				continue;
+			}
+
+			if ((r = ss_skb_cutoff_data(hm->msg.skb_head, c, 0,
+						    tfw_str_eolen(c))))
+				return r;
+			tfw_str_del_chunk(dup, id);
+			--c;
+			c_end = dup->chunks + dup->nchunks;
+		}
+	};
+
+	return 0;
+}
+
 /**
  * Substitute header value.
  *
@@ -1098,16 +1132,48 @@ tfw_http_msg_del_hbh_hdrs(TfwHttpMsg *hm)
 
 	do {
 		hid--;
-		if (hid == TFW_HTTP_HDR_CONNECTION)
+		if (hid == TFW_HTTP_HDR_CONNECTION
+		    && !(ht->tbl[hid].flags & TFW_STR_TRAILER))
 			continue;
-		if (ht->tbl[hid].flags & TFW_STR_HBH_HDR)
+		if (ht->tbl[hid].flags & TFW_STR_HBH_HDR) {
 			if ((r = __hdr_del(hm, hid)))
 				return r;
+		}
 	} while (hid);
 
 	return 0;
 }
 
+int
+tfw_http_msg_adjust_trailer_hdr(TfwHttpMsg *hm)
+{
+	TfwHttpHdrTbl *ht = hm->h_tbl;
+	int r = 0;
+
+	if (unlikely(hm->trailer_hid)) {
+		TfwStr *hdr = &ht->tbl[hm->trailer_hid];
+
+		/*
+		 * We mark 'Trailer' header with TFW_STR_TRAILER_HDR_HBP
+		 * if there is some hop by hop headers in it. And we
+		 * mark 'Trailer' header with TFW_STR_TRAILER_NOT_HDR_HBP
+		 * flag if there is some not hbp header in it.
+		 */
+		if (unlikely(hdr->flags & TFW_STR_TRAILER_HDR_HBP)) {
+			if (unlikely(hdr->flags &
+				     TFW_STR_TRAILER_NOT_HDR_HBP)) {
+				r = __hdr_del_part(hm, hm->trailer_hid,
+						   TFW_STR_TRAILER_HDR_HBP);
+			} else {
+				r = __hdr_del(hm, hm->trailer_hid);
+			}
+
+		}
+	}
+
+	return r;
+}
+
 /**
  * Remove flagged data and EOL from skb of TfwHttpMsg->body.
  *

fw/http_msg.h

@@ -159,6 +159,8 @@ int tfw_http_msg_hdr_xfrm(TfwHttpMsg *hm, char *name, size_t n_len,
 
 int tfw_http_msg_del_str(TfwHttpMsg *hm, TfwStr *str);
 int tfw_http_msg_del_hbh_hdrs(TfwHttpMsg *hm);
+int tfw_http_msg_adjust_trailer_hdr(TfwHttpMsg *hm);
+
 int tfw_http_msg_cutoff_body_chunks(TfwHttpResp *resp);
 
 int tfw_http_msg_setup(TfwHttpMsg *hm, TfwMsgIter *it, size_t data_len,