1aa97dd1cSKonstantin Ananyev/*
27e18fa1bSKonstantin Ananyev * Copyright (c) 2016-2017  Intel Corporation.
3aa97dd1cSKonstantin Ananyev * Licensed under the Apache License, Version 2.0 (the "License");
4aa97dd1cSKonstantin Ananyev * you may not use this file except in compliance with the License.
5aa97dd1cSKonstantin Ananyev * You may obtain a copy of the License at:
6aa97dd1cSKonstantin Ananyev *
7aa97dd1cSKonstantin Ananyev *     http://www.apache.org/licenses/LICENSE-2.0
8aa97dd1cSKonstantin Ananyev *
9aa97dd1cSKonstantin Ananyev * Unless required by applicable law or agreed to in writing, software
10aa97dd1cSKonstantin Ananyev * distributed under the License is distributed on an "AS IS" BASIS,
11aa97dd1cSKonstantin Ananyev * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12aa97dd1cSKonstantin Ananyev * See the License for the specific language governing permissions and
13aa97dd1cSKonstantin Ananyev * limitations under the License.
14aa97dd1cSKonstantin Ananyev */
15aa97dd1cSKonstantin Ananyev
16aa97dd1cSKonstantin Ananyev#include <rte_errno.h>
17aa97dd1cSKonstantin Ananyev#include <rte_ethdev.h>
18aa97dd1cSKonstantin Ananyev#include <rte_ip.h>
19aa97dd1cSKonstantin Ananyev#include <rte_ip_frag.h>
20aa97dd1cSKonstantin Ananyev#include <rte_tcp.h>
21aa97dd1cSKonstantin Ananyev
22aa97dd1cSKonstantin Ananyev#include "tcp_stream.h"
23aa97dd1cSKonstantin Ananyev#include "tcp_timer.h"
24aa97dd1cSKonstantin Ananyev#include "stream_table.h"
25aa97dd1cSKonstantin Ananyev#include "syncookie.h"
26aa97dd1cSKonstantin Ananyev#include "misc.h"
27aa97dd1cSKonstantin Ananyev#include "tcp_ctl.h"
28aa97dd1cSKonstantin Ananyev#include "tcp_rxq.h"
29aa97dd1cSKonstantin Ananyev#include "tcp_txq.h"
30c4c44906SMohammad Abdul Awal#include "tcp_tx_seg.h"
31aa97dd1cSKonstantin Ananyev
32aa97dd1cSKonstantin Ananyev#define	TCP_MAX_PKT_SEG	0x20
33aa97dd1cSKonstantin Ananyev
34aa97dd1cSKonstantin Ananyev/*
35aa97dd1cSKonstantin Ananyev * checks if input TCP ports and IP addresses match given stream.
36aa97dd1cSKonstantin Ananyev * returns zero on success.
37aa97dd1cSKonstantin Ananyev */
38aa97dd1cSKonstantin Ananyevstatic inline int
39aa97dd1cSKonstantin Ananyevrx_check_stream(const struct tle_tcp_stream *s, const union pkt_info *pi)
40aa97dd1cSKonstantin Ananyev{
41aa97dd1cSKonstantin Ananyev	int32_t rc;
42aa97dd1cSKonstantin Ananyev
43aa97dd1cSKonstantin Ananyev	if (pi->tf.type == TLE_V4)
44aa97dd1cSKonstantin Ananyev		rc = (pi->port.raw & s->s.pmsk.raw) != s->s.port.raw ||
45aa97dd1cSKonstantin Ananyev			(pi->addr4.raw & s->s.ipv4.mask.raw) !=
46aa97dd1cSKonstantin Ananyev			s->s.ipv4.addr.raw;
47aa97dd1cSKonstantin Ananyev	else
48aa97dd1cSKonstantin Ananyev		rc = (pi->port.raw & s->s.pmsk.raw) != s->s.port.raw ||
49aa97dd1cSKonstantin Ananyev			ymm_mask_cmp(&pi->addr6->raw, &s->s.ipv6.addr.raw,
50aa97dd1cSKonstantin Ananyev			&s->s.ipv6.mask.raw) != 0;
51aa97dd1cSKonstantin Ananyev
52aa97dd1cSKonstantin Ananyev	return rc;
53aa97dd1cSKonstantin Ananyev}
54aa97dd1cSKonstantin Ananyev
55aa97dd1cSKonstantin Ananyevstatic inline struct tle_tcp_stream *
56aa97dd1cSKonstantin Ananyevrx_obtain_listen_stream(const struct tle_dev *dev, const union pkt_info *pi,
57aa97dd1cSKonstantin Ananyev	uint32_t type)
58aa97dd1cSKonstantin Ananyev{
59aa97dd1cSKonstantin Ananyev	struct tle_tcp_stream *s;
60aa97dd1cSKonstantin Ananyev
61aa97dd1cSKonstantin Ananyev	s = (struct tle_tcp_stream *)dev->dp[type]->streams[pi->port.dst];
627e18fa1bSKonstantin Ananyev	if (s == NULL || tcp_stream_acquire(s) < 0)
63aa97dd1cSKonstantin Ananyev		return NULL;
64aa97dd1cSKonstantin Ananyev
65aa97dd1cSKonstantin Ananyev	/* check that we have a proper stream. */
66aa97dd1cSKonstantin Ananyev	if (s->tcb.state != TCP_ST_LISTEN) {
677e18fa1bSKonstantin Ananyev		tcp_stream_release(s);
68aa97dd1cSKonstantin Ananyev		s = NULL;
69aa97dd1cSKonstantin Ananyev	}
70aa97dd1cSKonstantin Ananyev
71aa97dd1cSKonstantin Ananyev	return s;
72aa97dd1cSKonstantin Ananyev}
73aa97dd1cSKonstantin Ananyev
74aa97dd1cSKonstantin Ananyevstatic inline struct tle_tcp_stream *
75aa97dd1cSKonstantin Ananyevrx_obtain_stream(const struct tle_dev *dev, struct stbl *st,
76aa97dd1cSKonstantin Ananyev	const union pkt_info *pi, uint32_t type)
77aa97dd1cSKonstantin Ananyev{
78aa97dd1cSKonstantin Ananyev	struct tle_tcp_stream *s;
79aa97dd1cSKonstantin Ananyev
80aa97dd1cSKonstantin Ananyev	s = stbl_find_data(st, pi);
81aa97dd1cSKonstantin Ananyev	if (s == NULL) {
82aa97dd1cSKonstantin Ananyev		if (pi->tf.flags == TCP_FLAG_ACK)
83aa97dd1cSKonstantin Ananyev			return rx_obtain_listen_stream(dev, pi, type);
84aa97dd1cSKonstantin Ananyev		return NULL;
85aa97dd1cSKonstantin Ananyev	}
86aa97dd1cSKonstantin Ananyev
877e18fa1bSKonstantin Ananyev	if (tcp_stream_acquire(s) < 0)
88aa97dd1cSKonstantin Ananyev		return NULL;
89aa97dd1cSKonstantin Ananyev	/* check that we have a proper stream. */
90aa97dd1cSKonstantin Ananyev	else if (s->tcb.state == TCP_ST_CLOSED) {
917e18fa1bSKonstantin Ananyev		tcp_stream_release(s);
92aa97dd1cSKonstantin Ananyev		s = NULL;
93aa97dd1cSKonstantin Ananyev	}
94aa97dd1cSKonstantin Ananyev
95aa97dd1cSKonstantin Ananyev	return s;
96aa97dd1cSKonstantin Ananyev}
97aa97dd1cSKonstantin Ananyev
98aa97dd1cSKonstantin Ananyev/*
99aa97dd1cSKonstantin Ananyev * Consider 2 pkt_info *equal* if their:
100aa97dd1cSKonstantin Ananyev * - types (IPv4/IPv6)
101aa97dd1cSKonstantin Ananyev * - TCP flags
102aa97dd1cSKonstantin Ananyev * - checksum flags
103aa97dd1cSKonstantin Ananyev * - TCP src and dst ports
104aa97dd1cSKonstantin Ananyev * - IP src and dst addresses
105aa97dd1cSKonstantin Ananyev * are equal.
106aa97dd1cSKonstantin Ananyev */
107aa97dd1cSKonstantin Ananyevstatic inline int
108aa97dd1cSKonstantin Ananyevpkt_info_bulk_eq(const union pkt_info pi[], uint32_t num)
109aa97dd1cSKonstantin Ananyev{
110aa97dd1cSKonstantin Ananyev	uint32_t i;
111aa97dd1cSKonstantin Ananyev
112aa97dd1cSKonstantin Ananyev	i = 1;
113aa97dd1cSKonstantin Ananyev
114aa97dd1cSKonstantin Ananyev	if (pi[0].tf.type == TLE_V4) {
115aa97dd1cSKonstantin Ananyev		while (i != num && xmm_cmp(&pi[0].raw, &pi[i].raw) == 0)
116aa97dd1cSKonstantin Ananyev			i++;
117aa97dd1cSKonstantin Ananyev
118aa97dd1cSKonstantin Ananyev	} else if (pi[0].tf.type == TLE_V6) {
119aa97dd1cSKonstantin Ananyev		while (i != num &&
120aa97dd1cSKonstantin Ananyev				pi[0].raw.u64[0] == pi[i].raw.u64[0] &&
121aa97dd1cSKonstantin Ananyev				ymm_cmp(&pi[0].addr6->raw,
122aa97dd1cSKonstantin Ananyev				&pi[i].addr6->raw) == 0)
123aa97dd1cSKonstantin Ananyev			i++;
124aa97dd1cSKonstantin Ananyev	}
125aa97dd1cSKonstantin Ananyev
126aa97dd1cSKonstantin Ananyev	return i;
127aa97dd1cSKonstantin Ananyev}
128aa97dd1cSKonstantin Ananyev
129aa97dd1cSKonstantin Ananyevstatic inline int
130aa97dd1cSKonstantin Ananyevpkt_info_bulk_syneq(const union pkt_info pi[], uint32_t num)
131aa97dd1cSKonstantin Ananyev{
132aa97dd1cSKonstantin Ananyev	uint32_t i;
133aa97dd1cSKonstantin Ananyev
134aa97dd1cSKonstantin Ananyev	i = 1;
135aa97dd1cSKonstantin Ananyev
136aa97dd1cSKonstantin Ananyev	if (pi[0].tf.type == TLE_V4) {
137aa97dd1cSKonstantin Ananyev		while (i != num && pi[0].tf.raw == pi[i].tf.raw &&
138aa97dd1cSKonstantin Ananyev				pi[0].port.dst == pi[i].port.dst &&
139aa97dd1cSKonstantin Ananyev				pi[0].addr4.dst == pi[i].addr4.dst)
140aa97dd1cSKonstantin Ananyev			i++;
141aa97dd1cSKonstantin Ananyev
142aa97dd1cSKonstantin Ananyev	} else if (pi[0].tf.type == TLE_V6) {
143aa97dd1cSKonstantin Ananyev		while (i != num && pi[0].tf.raw == pi[i].tf.raw &&
144aa97dd1cSKonstantin Ananyev				pi[0].port.dst == pi[i].port.dst &&
145aa97dd1cSKonstantin Ananyev				xmm_cmp(&pi[0].addr6->dst,
146aa97dd1cSKonstantin Ananyev				&pi[i].addr6->dst) == 0)
147aa97dd1cSKonstantin Ananyev			i++;
148aa97dd1cSKonstantin Ananyev	}
149aa97dd1cSKonstantin Ananyev
150aa97dd1cSKonstantin Ananyev	return i;
151aa97dd1cSKonstantin Ananyev}
152aa97dd1cSKonstantin Ananyev
153aa97dd1cSKonstantin Ananyevstatic inline void
154aa97dd1cSKonstantin Ananyevstream_drb_free(struct tle_tcp_stream *s, struct tle_drb *drbs[],
155aa97dd1cSKonstantin Ananyev	uint32_t nb_drb)
156aa97dd1cSKonstantin Ananyev{
157fbba0a3bSMohammad Abdul Awal	_rte_ring_enqueue_burst(s->tx.drb.r, (void **)drbs, nb_drb);
158aa97dd1cSKonstantin Ananyev}
159aa97dd1cSKonstantin Ananyev
160aa97dd1cSKonstantin Ananyevstatic inline uint32_t
161aa97dd1cSKonstantin Ananyevstream_drb_alloc(struct tle_tcp_stream *s, struct tle_drb *drbs[],
162aa97dd1cSKonstantin Ananyev	uint32_t nb_drb)
163aa97dd1cSKonstantin Ananyev{
164fbba0a3bSMohammad Abdul Awal	return _rte_ring_dequeue_burst(s->tx.drb.r, (void **)drbs, nb_drb);
165aa97dd1cSKonstantin Ananyev}
166aa97dd1cSKonstantin Ananyev
1677e18fa1bSKonstantin Ananyevstatic inline uint32_t
1687e18fa1bSKonstantin Ananyevget_ip_pid(struct tle_dev *dev, uint32_t num, uint32_t type, uint32_t st)
1697e18fa1bSKonstantin Ananyev{
1707e18fa1bSKonstantin Ananyev	uint32_t pid;
1717e18fa1bSKonstantin Ananyev	rte_atomic32_t *pa;
1727e18fa1bSKonstantin Ananyev
1737e18fa1bSKonstantin Ananyev	pa = &dev->tx.packet_id[type];
1747e18fa1bSKonstantin Ananyev
1757e18fa1bSKonstantin Ananyev	if (st == 0) {
1767e18fa1bSKonstantin Ananyev		pid = rte_atomic32_add_return(pa, num);
1777e18fa1bSKonstantin Ananyev		return pid - num;
1787e18fa1bSKonstantin Ananyev	} else {
1797e18fa1bSKonstantin Ananyev		pid = rte_atomic32_read(pa);
1807e18fa1bSKonstantin Ananyev		rte_atomic32_set(pa, pid + num);
1817e18fa1bSKonstantin Ananyev		return pid;
1827e18fa1bSKonstantin Ananyev	}
1837e18fa1bSKonstantin Ananyev}
1847e18fa1bSKonstantin Ananyev
185aa97dd1cSKonstantin Ananyevstatic inline void
186aa97dd1cSKonstantin Ananyevfill_tcph(struct tcp_hdr *l4h, const struct tcb *tcb, union l4_ports port,
187aa97dd1cSKonstantin Ananyev	uint32_t seq, uint8_t hlen, uint8_t flags)
188aa97dd1cSKonstantin Ananyev{
189aa97dd1cSKonstantin Ananyev	uint16_t wnd;
190aa97dd1cSKonstantin Ananyev
191aa97dd1cSKonstantin Ananyev	l4h->src_port = port.dst;
192aa97dd1cSKonstantin Ananyev	l4h->dst_port = port.src;
193aa97dd1cSKonstantin Ananyev
194aa97dd1cSKonstantin Ananyev	wnd = (flags & TCP_FLAG_SYN) ?
19521e7392fSKonstantin Ananyev		RTE_MIN(tcb->rcv.wnd, (uint32_t)UINT16_MAX) :
196aa97dd1cSKonstantin Ananyev		tcb->rcv.wnd >> tcb->rcv.wscale;
197aa97dd1cSKonstantin Ananyev
198aa97dd1cSKonstantin Ananyev	/* ??? use sse shuffle to hton all remaining 16 bytes at once. ??? */
199aa97dd1cSKonstantin Ananyev	l4h->sent_seq = rte_cpu_to_be_32(seq);
200aa97dd1cSKonstantin Ananyev	l4h->recv_ack = rte_cpu_to_be_32(tcb->rcv.nxt);
201aa97dd1cSKonstantin Ananyev	l4h->data_off = hlen / TCP_DATA_ALIGN << TCP_DATA_OFFSET;
202aa97dd1cSKonstantin Ananyev	l4h->tcp_flags = flags;
203aa97dd1cSKonstantin Ananyev	l4h->rx_win = rte_cpu_to_be_16(wnd);
204aa97dd1cSKonstantin Ananyev	l4h->cksum = 0;
205aa97dd1cSKonstantin Ananyev	l4h->tcp_urp = 0;
206aa97dd1cSKonstantin Ananyev
207aa97dd1cSKonstantin Ananyev	if (flags & TCP_FLAG_SYN)
208aa97dd1cSKonstantin Ananyev		fill_syn_opts(l4h + 1, &tcb->so);
209aa97dd1cSKonstantin Ananyev	else if ((flags & TCP_FLAG_RST) == 0 && tcb->so.ts.raw != 0)
210aa97dd1cSKonstantin Ananyev		fill_tms_opts(l4h + 1, tcb->snd.ts, tcb->rcv.ts);
211aa97dd1cSKonstantin Ananyev}
212aa97dd1cSKonstantin Ananyev
213aa97dd1cSKonstantin Ananyevstatic inline int
214aa97dd1cSKonstantin Ananyevtcp_fill_mbuf(struct rte_mbuf *m, const struct tle_tcp_stream *s,
215aa97dd1cSKonstantin Ananyev	const struct tle_dest *dst, uint64_t ol_flags,
216aa97dd1cSKonstantin Ananyev	union l4_ports port, uint32_t seq, uint32_t flags,
217aa97dd1cSKonstantin Ananyev	uint32_t pid, uint32_t swcsm)
218aa97dd1cSKonstantin Ananyev{
219aa97dd1cSKonstantin Ananyev	uint32_t l4, len, plen;
220aa97dd1cSKonstantin Ananyev	struct tcp_hdr *l4h;
221aa97dd1cSKonstantin Ananyev	char *l2h;
222aa97dd1cSKonstantin Ananyev
223aa97dd1cSKonstantin Ananyev	len = dst->l2_len + dst->l3_len;
224aa97dd1cSKonstantin Ananyev	plen = m->pkt_len;
225aa97dd1cSKonstantin Ananyev
226aa97dd1cSKonstantin Ananyev	if (flags & TCP_FLAG_SYN)
227aa97dd1cSKonstantin Ananyev		l4 = sizeof(*l4h) + TCP_TX_OPT_LEN_MAX;
228aa97dd1cSKonstantin Ananyev	else if ((flags & TCP_FLAG_RST) == 0 && s->tcb.rcv.ts != 0)
229aa97dd1cSKonstantin Ananyev		l4 = sizeof(*l4h) + TCP_TX_OPT_LEN_TMS;
230aa97dd1cSKonstantin Ananyev	else
231aa97dd1cSKonstantin Ananyev		l4 = sizeof(*l4h);
232aa97dd1cSKonstantin Ananyev
233aa97dd1cSKonstantin Ananyev	/* adjust mbuf to put L2/L3/L4 headers into it. */
234aa97dd1cSKonstantin Ananyev	l2h = rte_pktmbuf_prepend(m, len + l4);
235aa97dd1cSKonstantin Ananyev	if (l2h == NULL)
236aa97dd1cSKonstantin Ananyev		return -EINVAL;
237aa97dd1cSKonstantin Ananyev
238aa97dd1cSKonstantin Ananyev	/* copy L2/L3 header */
239aa97dd1cSKonstantin Ananyev	rte_memcpy(l2h, dst->hdr, len);
240aa97dd1cSKonstantin Ananyev
241aa97dd1cSKonstantin Ananyev	/* setup TCP header & options */
242aa97dd1cSKonstantin Ananyev	l4h = (struct tcp_hdr *)(l2h + len);
243aa97dd1cSKonstantin Ananyev	fill_tcph(l4h, &s->tcb, port, seq, l4, flags);
244aa97dd1cSKonstantin Ananyev
245aa97dd1cSKonstantin Ananyev	/* setup mbuf TX offload related fields. */
246aa97dd1cSKonstantin Ananyev	m->tx_offload = _mbuf_tx_offload(dst->l2_len, dst->l3_len, l4, 0, 0, 0);
247aa97dd1cSKonstantin Ananyev	m->ol_flags |= ol_flags;
248aa97dd1cSKonstantin Ananyev
249aa97dd1cSKonstantin Ananyev	/* update proto specific fields. */
250aa97dd1cSKonstantin Ananyev
251aa97dd1cSKonstantin Ananyev	if (s->s.type == TLE_V4) {
252aa97dd1cSKonstantin Ananyev		struct ipv4_hdr *l3h;
253aa97dd1cSKonstantin Ananyev		l3h = (struct ipv4_hdr *)(l2h + dst->l2_len);
254aa97dd1cSKonstantin Ananyev		l3h->packet_id = rte_cpu_to_be_16(pid);
255aa97dd1cSKonstantin Ananyev		l3h->total_length = rte_cpu_to_be_16(plen + dst->l3_len + l4);
256aa97dd1cSKonstantin Ananyev
257aa97dd1cSKonstantin Ananyev		if ((ol_flags & PKT_TX_TCP_CKSUM) != 0)
258aa97dd1cSKonstantin Ananyev			l4h->cksum = _ipv4x_phdr_cksum(l3h, m->l3_len,
259aa97dd1cSKonstantin Ananyev				ol_flags);
260aa97dd1cSKonstantin Ananyev		else if (swcsm != 0)
261aa97dd1cSKonstantin Ananyev			l4h->cksum = _ipv4_udptcp_mbuf_cksum(m, len, l3h);
262aa97dd1cSKonstantin Ananyev
263aa97dd1cSKonstantin Ananyev		if ((ol_flags & PKT_TX_IP_CKSUM) == 0 && swcsm != 0)
264aa97dd1cSKonstantin Ananyev			l3h->hdr_checksum = _ipv4x_cksum(l3h, m->l3_len);
265aa97dd1cSKonstantin Ananyev	} else {
266aa97dd1cSKonstantin Ananyev		struct ipv6_hdr *l3h;
267aa97dd1cSKonstantin Ananyev		l3h = (struct ipv6_hdr *)(l2h + dst->l2_len);
268aa97dd1cSKonstantin Ananyev		l3h->payload_len = rte_cpu_to_be_16(plen + l4);
269aa97dd1cSKonstantin Ananyev		if ((ol_flags & PKT_TX_TCP_CKSUM) != 0)
270aa97dd1cSKonstantin Ananyev			l4h->cksum = rte_ipv6_phdr_cksum(l3h, ol_flags);
271aa97dd1cSKonstantin Ananyev		else if (swcsm != 0)
272aa97dd1cSKonstantin Ananyev			l4h->cksum = _ipv6_udptcp_mbuf_cksum(m, len, l3h);
273aa97dd1cSKonstantin Ananyev	}
274aa97dd1cSKonstantin Ananyev
275aa97dd1cSKonstantin Ananyev	return 0;
276aa97dd1cSKonstantin Ananyev}
277aa97dd1cSKonstantin Ananyev
278aa97dd1cSKonstantin Ananyev/*
279aa97dd1cSKonstantin Ananyev * That function supposed to be used only for data packets.
280aa97dd1cSKonstantin Ananyev * Assumes that L2/L3/L4 headers and mbuf fields already setup properly.
281aa97dd1cSKonstantin Ananyev *  - updates tcp SEG.SEQ, SEG.ACK, TS.VAL, TS.ECR.
282aa97dd1cSKonstantin Ananyev *  - if no HW cksum offloads are enabled, calculates TCP checksum.
283aa97dd1cSKonstantin Ananyev */
284aa97dd1cSKonstantin Ananyevstatic inline void
285aa97dd1cSKonstantin Ananyevtcp_update_mbuf(struct rte_mbuf *m, uint32_t type, const struct tcb *tcb,
286aa97dd1cSKonstantin Ananyev	uint32_t seq, uint32_t pid)
287aa97dd1cSKonstantin Ananyev{
288aa97dd1cSKonstantin Ananyev	struct tcp_hdr *l4h;
289aa97dd1cSKonstantin Ananyev	uint32_t len;
290aa97dd1cSKonstantin Ananyev
291aa97dd1cSKonstantin Ananyev	len = m->l2_len + m->l3_len;
292aa97dd1cSKonstantin Ananyev	l4h = rte_pktmbuf_mtod_offset(m, struct tcp_hdr *, len);
293aa97dd1cSKonstantin Ananyev
294aa97dd1cSKonstantin Ananyev	l4h->sent_seq = rte_cpu_to_be_32(seq);
295aa97dd1cSKonstantin Ananyev	l4h->recv_ack = rte_cpu_to_be_32(tcb->rcv.nxt);
296aa97dd1cSKonstantin Ananyev
297aa97dd1cSKonstantin Ananyev	if (tcb->so.ts.raw != 0)
298aa97dd1cSKonstantin Ananyev		fill_tms_opts(l4h + 1, tcb->snd.ts, tcb->rcv.ts);
299aa97dd1cSKonstantin Ananyev
300aa97dd1cSKonstantin Ananyev	if (type == TLE_V4) {
301aa97dd1cSKonstantin Ananyev		struct ipv4_hdr *l3h;
302aa97dd1cSKonstantin Ananyev		l3h = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *, m->l2_len);
303aa97dd1cSKonstantin Ananyev		l3h->hdr_checksum = 0;
304aa97dd1cSKonstantin Ananyev		l3h->packet_id = rte_cpu_to_be_16(pid);
305aa97dd1cSKonstantin Ananyev		if ((m->ol_flags & PKT_TX_IP_CKSUM) == 0)
306aa97dd1cSKonstantin Ananyev			l3h->hdr_checksum = _ipv4x_cksum(l3h, m->l3_len);
307aa97dd1cSKonstantin Ananyev	}
308aa97dd1cSKonstantin Ananyev
309aa97dd1cSKonstantin Ananyev	/* have to calculate TCP checksum in SW */
310aa97dd1cSKonstantin Ananyev	if ((m->ol_flags & PKT_TX_TCP_CKSUM) == 0) {
311aa97dd1cSKonstantin Ananyev
312aa97dd1cSKonstantin Ananyev		l4h->cksum = 0;
313aa97dd1cSKonstantin Ananyev
314aa97dd1cSKonstantin Ananyev		if (type == TLE_V4) {
315aa97dd1cSKonstantin Ananyev			struct ipv4_hdr *l3h;
316aa97dd1cSKonstantin Ananyev			l3h = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *,
317aa97dd1cSKonstantin Ananyev				m->l2_len);
318aa97dd1cSKonstantin Ananyev			l4h->cksum = _ipv4_udptcp_mbuf_cksum(m, len, l3h);
319aa97dd1cSKonstantin Ananyev
320aa97dd1cSKonstantin Ananyev		} else {
321aa97dd1cSKonstantin Ananyev			struct ipv6_hdr *l3h;
322aa97dd1cSKonstantin Ananyev			l3h = rte_pktmbuf_mtod_offset(m, struct ipv6_hdr *,
323aa97dd1cSKonstantin Ananyev				m->l2_len);
324aa97dd1cSKonstantin Ananyev			l4h->cksum = _ipv6_udptcp_mbuf_cksum(m, len, l3h);
325aa97dd1cSKonstantin Ananyev		}
326aa97dd1cSKonstantin Ananyev	}
327aa97dd1cSKonstantin Ananyev}
328aa97dd1cSKonstantin Ananyev
329aa97dd1cSKonstantin Ananyev/* Send data packets that need to be ACK-ed by peer */
330aa97dd1cSKonstantin Ananyevstatic inline uint32_t
331aa97dd1cSKonstantin Ananyevtx_data_pkts(struct tle_tcp_stream *s, struct rte_mbuf *const m[], uint32_t num)
332aa97dd1cSKonstantin Ananyev{
333aa97dd1cSKonstantin Ananyev	uint32_t bsz, i, nb, nbm;
334aa97dd1cSKonstantin Ananyev	struct tle_dev *dev;
335aa97dd1cSKonstantin Ananyev	struct tle_drb *drb[num];
336aa97dd1cSKonstantin Ananyev
337aa97dd1cSKonstantin Ananyev	/* calculate how many drbs are needed.*/
338aa97dd1cSKonstantin Ananyev	bsz = s->tx.drb.nb_elem;
339aa97dd1cSKonstantin Ananyev	nbm = (num + bsz - 1) / bsz;
340aa97dd1cSKonstantin Ananyev
341aa97dd1cSKonstantin Ananyev	/* allocate drbs, adjust number of packets. */
342aa97dd1cSKonstantin Ananyev	nb = stream_drb_alloc(s, drb, nbm);
343aa97dd1cSKonstantin Ananyev
344aa97dd1cSKonstantin Ananyev	/* drb ring is empty. */
345aa97dd1cSKonstantin Ananyev	if (nb == 0)
346aa97dd1cSKonstantin Ananyev		return 0;
347aa97dd1cSKonstantin Ananyev
348aa97dd1cSKonstantin Ananyev	else if (nb != nbm)
349aa97dd1cSKonstantin Ananyev		num = nb * bsz;
350aa97dd1cSKonstantin Ananyev
351aa97dd1cSKonstantin Ananyev	dev = s->tx.dst.dev;
352aa97dd1cSKonstantin Ananyev
353aa97dd1cSKonstantin Ananyev	/* enqueue pkts for TX. */
354aa97dd1cSKonstantin Ananyev	nbm = nb;
355aa97dd1cSKonstantin Ananyev	i = tle_dring_mp_enqueue(&dev->tx.dr, (const void * const*)m,
356aa97dd1cSKonstantin Ananyev		num, drb, &nb);
357aa97dd1cSKonstantin Ananyev
358aa97dd1cSKonstantin Ananyev	/* free unused drbs. */
359aa97dd1cSKonstantin Ananyev	if (nb != 0)
360aa97dd1cSKonstantin Ananyev		stream_drb_free(s, drb + nbm - nb, nb);
361aa97dd1cSKonstantin Ananyev
362aa97dd1cSKonstantin Ananyev	return i;
363aa97dd1cSKonstantin Ananyev}
364aa97dd1cSKonstantin Ananyev