tcp_rxtx.c revision aa97dd1c
1aa97dd1cSKonstantin Ananyev/*
2aa97dd1cSKonstantin Ananyev * Copyright (c) 2016  Intel Corporation.
3aa97dd1cSKonstantin Ananyev * Licensed under the Apache License, Version 2.0 (the "License");
4aa97dd1cSKonstantin Ananyev * you may not use this file except in compliance with the License.
5aa97dd1cSKonstantin Ananyev * You may obtain a copy of the License at:
6aa97dd1cSKonstantin Ananyev *
7aa97dd1cSKonstantin Ananyev *     http://www.apache.org/licenses/LICENSE-2.0
8aa97dd1cSKonstantin Ananyev *
9aa97dd1cSKonstantin Ananyev * Unless required by applicable law or agreed to in writing, software
10aa97dd1cSKonstantin Ananyev * distributed under the License is distributed on an "AS IS" BASIS,
11aa97dd1cSKonstantin Ananyev * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12aa97dd1cSKonstantin Ananyev * See the License for the specific language governing permissions and
13aa97dd1cSKonstantin Ananyev * limitations under the License.
14aa97dd1cSKonstantin Ananyev */
15aa97dd1cSKonstantin Ananyev
16aa97dd1cSKonstantin Ananyev#include <rte_errno.h>
17aa97dd1cSKonstantin Ananyev#include <rte_ethdev.h>
18aa97dd1cSKonstantin Ananyev#include <rte_ip.h>
19aa97dd1cSKonstantin Ananyev#include <rte_ip_frag.h>
20aa97dd1cSKonstantin Ananyev#include <rte_tcp.h>
21aa97dd1cSKonstantin Ananyev
22aa97dd1cSKonstantin Ananyev#include "tcp_stream.h"
23aa97dd1cSKonstantin Ananyev#include "tcp_timer.h"
24aa97dd1cSKonstantin Ananyev#include "stream_table.h"
25aa97dd1cSKonstantin Ananyev#include "syncookie.h"
26aa97dd1cSKonstantin Ananyev#include "misc.h"
27aa97dd1cSKonstantin Ananyev#include "tcp_ctl.h"
28aa97dd1cSKonstantin Ananyev#include "tcp_rxq.h"
29aa97dd1cSKonstantin Ananyev#include "tcp_txq.h"
30aa97dd1cSKonstantin Ananyev
31aa97dd1cSKonstantin Ananyev#define	TCP_MAX_PKT_SEG	0x20
32aa97dd1cSKonstantin Ananyev
33aa97dd1cSKonstantin Ananyev/*
34aa97dd1cSKonstantin Ananyev * checks if input TCP ports and IP addresses match given stream.
35aa97dd1cSKonstantin Ananyev * returns zero on success.
36aa97dd1cSKonstantin Ananyev */
37aa97dd1cSKonstantin Ananyevstatic inline int
38aa97dd1cSKonstantin Ananyevrx_check_stream(const struct tle_tcp_stream *s, const union pkt_info *pi)
39aa97dd1cSKonstantin Ananyev{
40aa97dd1cSKonstantin Ananyev	int32_t rc;
41aa97dd1cSKonstantin Ananyev
42aa97dd1cSKonstantin Ananyev	if (pi->tf.type == TLE_V4)
43aa97dd1cSKonstantin Ananyev		rc = (pi->port.raw & s->s.pmsk.raw) != s->s.port.raw ||
44aa97dd1cSKonstantin Ananyev			(pi->addr4.raw & s->s.ipv4.mask.raw) !=
45aa97dd1cSKonstantin Ananyev			s->s.ipv4.addr.raw;
46aa97dd1cSKonstantin Ananyev	else
47aa97dd1cSKonstantin Ananyev		rc = (pi->port.raw & s->s.pmsk.raw) != s->s.port.raw ||
48aa97dd1cSKonstantin Ananyev			ymm_mask_cmp(&pi->addr6->raw, &s->s.ipv6.addr.raw,
49aa97dd1cSKonstantin Ananyev			&s->s.ipv6.mask.raw) != 0;
50aa97dd1cSKonstantin Ananyev
51aa97dd1cSKonstantin Ananyev	return rc;
52aa97dd1cSKonstantin Ananyev}
53aa97dd1cSKonstantin Ananyev
54aa97dd1cSKonstantin Ananyevstatic inline struct tle_tcp_stream *
55aa97dd1cSKonstantin Ananyevrx_obtain_listen_stream(const struct tle_dev *dev, const union pkt_info *pi,
56aa97dd1cSKonstantin Ananyev	uint32_t type)
57aa97dd1cSKonstantin Ananyev{
58aa97dd1cSKonstantin Ananyev	struct tle_tcp_stream *s;
59aa97dd1cSKonstantin Ananyev
60aa97dd1cSKonstantin Ananyev	s = (struct tle_tcp_stream *)dev->dp[type]->streams[pi->port.dst];
61aa97dd1cSKonstantin Ananyev	if (s == NULL || rwl_acquire(&s->rx.use) < 0)
62aa97dd1cSKonstantin Ananyev		return NULL;
63aa97dd1cSKonstantin Ananyev
64aa97dd1cSKonstantin Ananyev	/* check that we have a proper stream. */
65aa97dd1cSKonstantin Ananyev	if (s->tcb.state != TCP_ST_LISTEN) {
66aa97dd1cSKonstantin Ananyev		rwl_release(&s->rx.use);
67aa97dd1cSKonstantin Ananyev		s = NULL;
68aa97dd1cSKonstantin Ananyev	}
69aa97dd1cSKonstantin Ananyev
70aa97dd1cSKonstantin Ananyev	return s;
71aa97dd1cSKonstantin Ananyev}
72aa97dd1cSKonstantin Ananyev
73aa97dd1cSKonstantin Ananyevstatic inline struct tle_tcp_stream *
74aa97dd1cSKonstantin Ananyevrx_obtain_stream(const struct tle_dev *dev, struct stbl *st,
75aa97dd1cSKonstantin Ananyev	const union pkt_info *pi, uint32_t type)
76aa97dd1cSKonstantin Ananyev{
77aa97dd1cSKonstantin Ananyev	struct tle_tcp_stream *s;
78aa97dd1cSKonstantin Ananyev
79aa97dd1cSKonstantin Ananyev	s = stbl_find_data(st, pi);
80aa97dd1cSKonstantin Ananyev	if (s == NULL) {
81aa97dd1cSKonstantin Ananyev		if (pi->tf.flags == TCP_FLAG_ACK)
82aa97dd1cSKonstantin Ananyev			return rx_obtain_listen_stream(dev, pi, type);
83aa97dd1cSKonstantin Ananyev		return NULL;
84aa97dd1cSKonstantin Ananyev	}
85aa97dd1cSKonstantin Ananyev
86aa97dd1cSKonstantin Ananyev	if (stbl_data_pkt(s) || rwl_acquire(&s->rx.use) < 0)
87aa97dd1cSKonstantin Ananyev		return NULL;
88aa97dd1cSKonstantin Ananyev	/* check that we have a proper stream. */
89aa97dd1cSKonstantin Ananyev	else if (s->tcb.state == TCP_ST_CLOSED) {
90aa97dd1cSKonstantin Ananyev		rwl_release(&s->rx.use);
91aa97dd1cSKonstantin Ananyev		s = NULL;
92aa97dd1cSKonstantin Ananyev	}
93aa97dd1cSKonstantin Ananyev
94aa97dd1cSKonstantin Ananyev	return s;
95aa97dd1cSKonstantin Ananyev}
96aa97dd1cSKonstantin Ananyev
97aa97dd1cSKonstantin Ananyev/*
98aa97dd1cSKonstantin Ananyev * Consider 2 pkt_info *equal* if their:
99aa97dd1cSKonstantin Ananyev * - types (IPv4/IPv6)
100aa97dd1cSKonstantin Ananyev * - TCP flags
101aa97dd1cSKonstantin Ananyev * - checksum flags
102aa97dd1cSKonstantin Ananyev * - TCP src and dst ports
103aa97dd1cSKonstantin Ananyev * - IP src and dst addresses
104aa97dd1cSKonstantin Ananyev * are equal.
105aa97dd1cSKonstantin Ananyev */
106aa97dd1cSKonstantin Ananyevstatic inline int
107aa97dd1cSKonstantin Ananyevpkt_info_bulk_eq(const union pkt_info pi[], uint32_t num)
108aa97dd1cSKonstantin Ananyev{
109aa97dd1cSKonstantin Ananyev	uint32_t i;
110aa97dd1cSKonstantin Ananyev
111aa97dd1cSKonstantin Ananyev	i = 1;
112aa97dd1cSKonstantin Ananyev
113aa97dd1cSKonstantin Ananyev	if (pi[0].tf.type == TLE_V4) {
114aa97dd1cSKonstantin Ananyev		while (i != num && xmm_cmp(&pi[0].raw, &pi[i].raw) == 0)
115aa97dd1cSKonstantin Ananyev			i++;
116aa97dd1cSKonstantin Ananyev
117aa97dd1cSKonstantin Ananyev	} else if (pi[0].tf.type == TLE_V6) {
118aa97dd1cSKonstantin Ananyev		while (i != num &&
119aa97dd1cSKonstantin Ananyev				pi[0].raw.u64[0] == pi[i].raw.u64[0] &&
120aa97dd1cSKonstantin Ananyev				ymm_cmp(&pi[0].addr6->raw,
121aa97dd1cSKonstantin Ananyev				&pi[i].addr6->raw) == 0)
122aa97dd1cSKonstantin Ananyev			i++;
123aa97dd1cSKonstantin Ananyev	}
124aa97dd1cSKonstantin Ananyev
125aa97dd1cSKonstantin Ananyev	return i;
126aa97dd1cSKonstantin Ananyev}
127aa97dd1cSKonstantin Ananyev
128aa97dd1cSKonstantin Ananyevstatic inline int
129aa97dd1cSKonstantin Ananyevpkt_info_bulk_syneq(const union pkt_info pi[], uint32_t num)
130aa97dd1cSKonstantin Ananyev{
131aa97dd1cSKonstantin Ananyev	uint32_t i;
132aa97dd1cSKonstantin Ananyev
133aa97dd1cSKonstantin Ananyev	i = 1;
134aa97dd1cSKonstantin Ananyev
135aa97dd1cSKonstantin Ananyev	if (pi[0].tf.type == TLE_V4) {
136aa97dd1cSKonstantin Ananyev		while (i != num && pi[0].tf.raw == pi[i].tf.raw &&
137aa97dd1cSKonstantin Ananyev				pi[0].port.dst == pi[i].port.dst &&
138aa97dd1cSKonstantin Ananyev				pi[0].addr4.dst == pi[i].addr4.dst)
139aa97dd1cSKonstantin Ananyev			i++;
140aa97dd1cSKonstantin Ananyev
141aa97dd1cSKonstantin Ananyev	} else if (pi[0].tf.type == TLE_V6) {
142aa97dd1cSKonstantin Ananyev		while (i != num && pi[0].tf.raw == pi[i].tf.raw &&
143aa97dd1cSKonstantin Ananyev				pi[0].port.dst == pi[i].port.dst &&
144aa97dd1cSKonstantin Ananyev				xmm_cmp(&pi[0].addr6->dst,
145aa97dd1cSKonstantin Ananyev				&pi[i].addr6->dst) == 0)
146aa97dd1cSKonstantin Ananyev			i++;
147aa97dd1cSKonstantin Ananyev	}
148aa97dd1cSKonstantin Ananyev
149aa97dd1cSKonstantin Ananyev	return i;
150aa97dd1cSKonstantin Ananyev}
151aa97dd1cSKonstantin Ananyev
152aa97dd1cSKonstantin Ananyevstatic inline void
153aa97dd1cSKonstantin Ananyevstream_drb_free(struct tle_tcp_stream *s, struct tle_drb *drbs[],
154aa97dd1cSKonstantin Ananyev	uint32_t nb_drb)
155aa97dd1cSKonstantin Ananyev{
156aa97dd1cSKonstantin Ananyev	rte_ring_enqueue_burst(s->tx.drb.r, (void **)drbs, nb_drb);
157aa97dd1cSKonstantin Ananyev}
158aa97dd1cSKonstantin Ananyev
159aa97dd1cSKonstantin Ananyevstatic inline uint32_t
160aa97dd1cSKonstantin Ananyevstream_drb_alloc(struct tle_tcp_stream *s, struct tle_drb *drbs[],
161aa97dd1cSKonstantin Ananyev	uint32_t nb_drb)
162aa97dd1cSKonstantin Ananyev{
163aa97dd1cSKonstantin Ananyev	return rte_ring_dequeue_burst(s->tx.drb.r, (void **)drbs, nb_drb);
164aa97dd1cSKonstantin Ananyev}
165aa97dd1cSKonstantin Ananyev
166aa97dd1cSKonstantin Ananyevstatic inline void
167aa97dd1cSKonstantin Ananyevfill_tcph(struct tcp_hdr *l4h, const struct tcb *tcb, union l4_ports port,
168aa97dd1cSKonstantin Ananyev	uint32_t seq, uint8_t hlen, uint8_t flags)
169aa97dd1cSKonstantin Ananyev{
170aa97dd1cSKonstantin Ananyev	uint16_t wnd;
171aa97dd1cSKonstantin Ananyev
172aa97dd1cSKonstantin Ananyev	l4h->src_port = port.dst;
173aa97dd1cSKonstantin Ananyev	l4h->dst_port = port.src;
174aa97dd1cSKonstantin Ananyev
175aa97dd1cSKonstantin Ananyev	wnd = (flags & TCP_FLAG_SYN) ?
176aa97dd1cSKonstantin Ananyev		RTE_MAX(TCP4_MIN_MSS, tcb->so.mss) :
177aa97dd1cSKonstantin Ananyev		tcb->rcv.wnd >> tcb->rcv.wscale;
178aa97dd1cSKonstantin Ananyev
179aa97dd1cSKonstantin Ananyev	/* ??? use sse shuffle to hton all remaining 16 bytes at once. ??? */
180aa97dd1cSKonstantin Ananyev	l4h->sent_seq = rte_cpu_to_be_32(seq);
181aa97dd1cSKonstantin Ananyev	l4h->recv_ack = rte_cpu_to_be_32(tcb->rcv.nxt);
182aa97dd1cSKonstantin Ananyev	l4h->data_off = hlen / TCP_DATA_ALIGN << TCP_DATA_OFFSET;
183aa97dd1cSKonstantin Ananyev	l4h->tcp_flags = flags;
184aa97dd1cSKonstantin Ananyev	l4h->rx_win = rte_cpu_to_be_16(wnd);
185aa97dd1cSKonstantin Ananyev	l4h->cksum = 0;
186aa97dd1cSKonstantin Ananyev	l4h->tcp_urp = 0;
187aa97dd1cSKonstantin Ananyev
188aa97dd1cSKonstantin Ananyev	if (flags & TCP_FLAG_SYN)
189aa97dd1cSKonstantin Ananyev		fill_syn_opts(l4h + 1, &tcb->so);
190aa97dd1cSKonstantin Ananyev	else if ((flags & TCP_FLAG_RST) == 0 && tcb->so.ts.raw != 0)
191aa97dd1cSKonstantin Ananyev		fill_tms_opts(l4h + 1, tcb->snd.ts, tcb->rcv.ts);
192aa97dd1cSKonstantin Ananyev}
193aa97dd1cSKonstantin Ananyev
194aa97dd1cSKonstantin Ananyevstatic inline int
195aa97dd1cSKonstantin Ananyevtcp_fill_mbuf(struct rte_mbuf *m, const struct tle_tcp_stream *s,
196aa97dd1cSKonstantin Ananyev	const struct tle_dest *dst, uint64_t ol_flags,
197aa97dd1cSKonstantin Ananyev	union l4_ports port, uint32_t seq, uint32_t flags,
198aa97dd1cSKonstantin Ananyev	uint32_t pid, uint32_t swcsm)
199aa97dd1cSKonstantin Ananyev{
200aa97dd1cSKonstantin Ananyev	uint32_t l4, len, plen;
201aa97dd1cSKonstantin Ananyev	struct tcp_hdr *l4h;
202aa97dd1cSKonstantin Ananyev	char *l2h;
203aa97dd1cSKonstantin Ananyev
204aa97dd1cSKonstantin Ananyev	len = dst->l2_len + dst->l3_len;
205aa97dd1cSKonstantin Ananyev	plen = m->pkt_len;
206aa97dd1cSKonstantin Ananyev
207aa97dd1cSKonstantin Ananyev	if (flags & TCP_FLAG_SYN)
208aa97dd1cSKonstantin Ananyev		l4 = sizeof(*l4h) + TCP_TX_OPT_LEN_MAX;
209aa97dd1cSKonstantin Ananyev	else if ((flags & TCP_FLAG_RST) == 0 && s->tcb.rcv.ts != 0)
210aa97dd1cSKonstantin Ananyev		l4 = sizeof(*l4h) + TCP_TX_OPT_LEN_TMS;
211aa97dd1cSKonstantin Ananyev	else
212aa97dd1cSKonstantin Ananyev		l4 = sizeof(*l4h);
213aa97dd1cSKonstantin Ananyev
214aa97dd1cSKonstantin Ananyev	/* adjust mbuf to put L2/L3/L4 headers into it. */
215aa97dd1cSKonstantin Ananyev	l2h = rte_pktmbuf_prepend(m, len + l4);
216aa97dd1cSKonstantin Ananyev	if (l2h == NULL)
217aa97dd1cSKonstantin Ananyev		return -EINVAL;
218aa97dd1cSKonstantin Ananyev
219aa97dd1cSKonstantin Ananyev	/* copy L2/L3 header */
220aa97dd1cSKonstantin Ananyev	rte_memcpy(l2h, dst->hdr, len);
221aa97dd1cSKonstantin Ananyev
222aa97dd1cSKonstantin Ananyev	/* setup TCP header & options */
223aa97dd1cSKonstantin Ananyev	l4h = (struct tcp_hdr *)(l2h + len);
224aa97dd1cSKonstantin Ananyev	fill_tcph(l4h, &s->tcb, port, seq, l4, flags);
225aa97dd1cSKonstantin Ananyev
226aa97dd1cSKonstantin Ananyev	/* setup mbuf TX offload related fields. */
227aa97dd1cSKonstantin Ananyev	m->tx_offload = _mbuf_tx_offload(dst->l2_len, dst->l3_len, l4, 0, 0, 0);
228aa97dd1cSKonstantin Ananyev	m->ol_flags |= ol_flags;
229aa97dd1cSKonstantin Ananyev
230aa97dd1cSKonstantin Ananyev	/* update proto specific fields. */
231aa97dd1cSKonstantin Ananyev
232aa97dd1cSKonstantin Ananyev	if (s->s.type == TLE_V4) {
233aa97dd1cSKonstantin Ananyev		struct ipv4_hdr *l3h;
234aa97dd1cSKonstantin Ananyev		l3h = (struct ipv4_hdr *)(l2h + dst->l2_len);
235aa97dd1cSKonstantin Ananyev		l3h->packet_id = rte_cpu_to_be_16(pid);
236aa97dd1cSKonstantin Ananyev		l3h->total_length = rte_cpu_to_be_16(plen + dst->l3_len + l4);
237aa97dd1cSKonstantin Ananyev
238aa97dd1cSKonstantin Ananyev		if ((ol_flags & PKT_TX_TCP_CKSUM) != 0)
239aa97dd1cSKonstantin Ananyev			l4h->cksum = _ipv4x_phdr_cksum(l3h, m->l3_len,
240aa97dd1cSKonstantin Ananyev				ol_flags);
241aa97dd1cSKonstantin Ananyev		else if (swcsm != 0)
242aa97dd1cSKonstantin Ananyev			l4h->cksum = _ipv4_udptcp_mbuf_cksum(m, len, l3h);
243aa97dd1cSKonstantin Ananyev
244aa97dd1cSKonstantin Ananyev		if ((ol_flags & PKT_TX_IP_CKSUM) == 0 && swcsm != 0)
245aa97dd1cSKonstantin Ananyev			l3h->hdr_checksum = _ipv4x_cksum(l3h, m->l3_len);
246aa97dd1cSKonstantin Ananyev	} else {
247aa97dd1cSKonstantin Ananyev		struct ipv6_hdr *l3h;
248aa97dd1cSKonstantin Ananyev		l3h = (struct ipv6_hdr *)(l2h + dst->l2_len);
249aa97dd1cSKonstantin Ananyev		l3h->payload_len = rte_cpu_to_be_16(plen + l4);
250aa97dd1cSKonstantin Ananyev		if ((ol_flags & PKT_TX_TCP_CKSUM) != 0)
251aa97dd1cSKonstantin Ananyev			l4h->cksum = rte_ipv6_phdr_cksum(l3h, ol_flags);
252aa97dd1cSKonstantin Ananyev		else if (swcsm != 0)
253aa97dd1cSKonstantin Ananyev			l4h->cksum = _ipv6_udptcp_mbuf_cksum(m, len, l3h);
254aa97dd1cSKonstantin Ananyev	}
255aa97dd1cSKonstantin Ananyev
256aa97dd1cSKonstantin Ananyev	return 0;
257aa97dd1cSKonstantin Ananyev}
258aa97dd1cSKonstantin Ananyev
259aa97dd1cSKonstantin Ananyev/*
260aa97dd1cSKonstantin Ananyev * That function supposed to be used only for data packets.
261aa97dd1cSKonstantin Ananyev * Assumes that L2/L3/L4 headers and mbuf fields already setup properly.
262aa97dd1cSKonstantin Ananyev *  - updates tcp SEG.SEQ, SEG.ACK, TS.VAL, TS.ECR.
263aa97dd1cSKonstantin Ananyev *  - if no HW cksum offloads are enabled, calculates TCP checksum.
264aa97dd1cSKonstantin Ananyev */
265aa97dd1cSKonstantin Ananyevstatic inline void
266aa97dd1cSKonstantin Ananyevtcp_update_mbuf(struct rte_mbuf *m, uint32_t type, const struct tcb *tcb,
267aa97dd1cSKonstantin Ananyev	uint32_t seq, uint32_t pid)
268aa97dd1cSKonstantin Ananyev{
269aa97dd1cSKonstantin Ananyev	struct tcp_hdr *l4h;
270aa97dd1cSKonstantin Ananyev	uint32_t len;
271aa97dd1cSKonstantin Ananyev
272aa97dd1cSKonstantin Ananyev	len = m->l2_len + m->l3_len;
273aa97dd1cSKonstantin Ananyev	l4h = rte_pktmbuf_mtod_offset(m, struct tcp_hdr *, len);
274aa97dd1cSKonstantin Ananyev
275aa97dd1cSKonstantin Ananyev	l4h->sent_seq = rte_cpu_to_be_32(seq);
276aa97dd1cSKonstantin Ananyev	l4h->recv_ack = rte_cpu_to_be_32(tcb->rcv.nxt);
277aa97dd1cSKonstantin Ananyev
278aa97dd1cSKonstantin Ananyev	if (tcb->so.ts.raw != 0)
279aa97dd1cSKonstantin Ananyev		fill_tms_opts(l4h + 1, tcb->snd.ts, tcb->rcv.ts);
280aa97dd1cSKonstantin Ananyev
281aa97dd1cSKonstantin Ananyev	if (type == TLE_V4) {
282aa97dd1cSKonstantin Ananyev		struct ipv4_hdr *l3h;
283aa97dd1cSKonstantin Ananyev		l3h = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *, m->l2_len);
284aa97dd1cSKonstantin Ananyev		l3h->hdr_checksum = 0;
285aa97dd1cSKonstantin Ananyev		l3h->packet_id = rte_cpu_to_be_16(pid);
286aa97dd1cSKonstantin Ananyev		if ((m->ol_flags & PKT_TX_IP_CKSUM) == 0)
287aa97dd1cSKonstantin Ananyev			l3h->hdr_checksum = _ipv4x_cksum(l3h, m->l3_len);
288aa97dd1cSKonstantin Ananyev	}
289aa97dd1cSKonstantin Ananyev
290aa97dd1cSKonstantin Ananyev	/* have to calculate TCP checksum in SW */
291aa97dd1cSKonstantin Ananyev	if ((m->ol_flags & PKT_TX_TCP_CKSUM) == 0) {
292aa97dd1cSKonstantin Ananyev
293aa97dd1cSKonstantin Ananyev		l4h->cksum = 0;
294aa97dd1cSKonstantin Ananyev
295aa97dd1cSKonstantin Ananyev		if (type == TLE_V4) {
296aa97dd1cSKonstantin Ananyev			struct ipv4_hdr *l3h;
297aa97dd1cSKonstantin Ananyev			l3h = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *,
298aa97dd1cSKonstantin Ananyev				m->l2_len);
299aa97dd1cSKonstantin Ananyev			l4h->cksum = _ipv4_udptcp_mbuf_cksum(m, len, l3h);
300aa97dd1cSKonstantin Ananyev
301aa97dd1cSKonstantin Ananyev		} else {
302aa97dd1cSKonstantin Ananyev			struct ipv6_hdr *l3h;
303aa97dd1cSKonstantin Ananyev			l3h = rte_pktmbuf_mtod_offset(m, struct ipv6_hdr *,
304aa97dd1cSKonstantin Ananyev				m->l2_len);
305aa97dd1cSKonstantin Ananyev			l4h->cksum = _ipv6_udptcp_mbuf_cksum(m, len, l3h);
306aa97dd1cSKonstantin Ananyev		}
307aa97dd1cSKonstantin Ananyev	}
308aa97dd1cSKonstantin Ananyev}
309aa97dd1cSKonstantin Ananyev
310aa97dd1cSKonstantin Ananyev/* Send data packets that need to be ACK-ed by peer */
311aa97dd1cSKonstantin Ananyevstatic inline uint32_t
312aa97dd1cSKonstantin Ananyevtx_data_pkts(struct tle_tcp_stream *s, struct rte_mbuf *const m[], uint32_t num)
313aa97dd1cSKonstantin Ananyev{
314aa97dd1cSKonstantin Ananyev	uint32_t bsz, i, nb, nbm;
315aa97dd1cSKonstantin Ananyev	struct tle_dev *dev;
316aa97dd1cSKonstantin Ananyev	struct tle_drb *drb[num];
317aa97dd1cSKonstantin Ananyev
318aa97dd1cSKonstantin Ananyev	/* calculate how many drbs are needed.*/
319aa97dd1cSKonstantin Ananyev	bsz = s->tx.drb.nb_elem;
320aa97dd1cSKonstantin Ananyev	nbm = (num + bsz - 1) / bsz;
321aa97dd1cSKonstantin Ananyev
322aa97dd1cSKonstantin Ananyev	/* allocate drbs, adjust number of packets. */
323aa97dd1cSKonstantin Ananyev	nb = stream_drb_alloc(s, drb, nbm);
324aa97dd1cSKonstantin Ananyev
325aa97dd1cSKonstantin Ananyev	/* drb ring is empty. */
326aa97dd1cSKonstantin Ananyev	if (nb == 0)
327aa97dd1cSKonstantin Ananyev		return 0;
328aa97dd1cSKonstantin Ananyev
329aa97dd1cSKonstantin Ananyev	else if (nb != nbm)
330aa97dd1cSKonstantin Ananyev		num = nb * bsz;
331aa97dd1cSKonstantin Ananyev
332aa97dd1cSKonstantin Ananyev	dev = s->tx.dst.dev;
333aa97dd1cSKonstantin Ananyev
334aa97dd1cSKonstantin Ananyev	/* enqueue pkts for TX. */
335aa97dd1cSKonstantin Ananyev	nbm = nb;
336aa97dd1cSKonstantin Ananyev	i = tle_dring_mp_enqueue(&dev->tx.dr, (const void * const*)m,
337aa97dd1cSKonstantin Ananyev		num, drb, &nb);
338aa97dd1cSKonstantin Ananyev
339aa97dd1cSKonstantin Ananyev	/* free unused drbs. */
340aa97dd1cSKonstantin Ananyev	if (nb != 0)
341aa97dd1cSKonstantin Ananyev		stream_drb_free(s, drb + nbm - nb, nb);
342aa97dd1cSKonstantin Ananyev
343aa97dd1cSKonstantin Ananyev	return i;
344aa97dd1cSKonstantin Ananyev}
345aa97dd1cSKonstantin Ananyev
346aa97dd1cSKonstantin Ananyevstatic inline uint32_t
347aa97dd1cSKonstantin Ananyevtx_data_bulk(struct tle_tcp_stream *s, union seqlen *sl, struct rte_mbuf *mi[],
348aa97dd1cSKonstantin Ananyev	uint32_t num)
349aa97dd1cSKonstantin Ananyev{
350aa97dd1cSKonstantin Ananyev	uint32_t fail, i, k, n, mss, pid, plen, sz, tn, type;
351aa97dd1cSKonstantin Ananyev	struct tle_dev *dev;
352aa97dd1cSKonstantin Ananyev	struct rte_mbuf *mb;
353aa97dd1cSKonstantin Ananyev	struct rte_mbuf *mo[MAX_PKT_BURST + TCP_MAX_PKT_SEG];
354aa97dd1cSKonstantin Ananyev
355aa97dd1cSKonstantin Ananyev	mss = s->tcb.snd.mss;
356aa97dd1cSKonstantin Ananyev	type = s->s.type;
357aa97dd1cSKonstantin Ananyev
358aa97dd1cSKonstantin Ananyev	dev = s->tx.dst.dev;
359aa97dd1cSKonstantin Ananyev	pid = rte_atomic32_add_return(&dev->tx.packet_id[type], num) - num;
360aa97dd1cSKonstantin Ananyev
361