pkt.c revision fbba0a3b
1/*
2 * Copyright (c) 2016  Intel Corporation.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 *     http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#include <netinet/ip6.h>
17#include <rte_arp.h>
18
19#include "netbe.h"
20
21struct ptype2cb {
22	uint32_t mask;
23	const char *name;
24	rte_rx_callback_fn fn;
25};
26
27enum {
28	ETHER_PTYPE = 0x1,
29	IPV4_PTYPE = 0x2,
30	IPV4_EXT_PTYPE = 0x4,
31	IPV6_PTYPE = 0x8,
32	IPV6_EXT_PTYPE = 0x10,
33	TCP_PTYPE = 0x20,
34	UDP_PTYPE = 0x40,
35};
36
37static inline uint64_t
38_mbuf_tx_offload(uint64_t il2, uint64_t il3, uint64_t il4, uint64_t tso,
39	uint64_t ol3, uint64_t ol2)
40{
41	return il2 | il3 << 7 | il4 << 16 | tso << 24 | ol3 << 40 | ol2 << 49;
42}
43
44static inline void
45fill_pkt_hdr_len(struct rte_mbuf *m, uint32_t l2, uint32_t l3, uint32_t l4)
46{
47	m->tx_offload = _mbuf_tx_offload(l2, l3, l4, 0, 0, 0);
48}
49
50static inline int
51is_ipv4_frag(const struct ipv4_hdr *iph)
52{
53	const uint16_t mask = rte_cpu_to_be_16(~IPV4_HDR_DF_FLAG);
54
55	return ((mask & iph->fragment_offset) != 0);
56}
57
58static inline uint32_t
59get_tcp_header_size(struct rte_mbuf *m, uint32_t l2_len, uint32_t l3_len)
60{
61	const struct tcp_hdr *tcp;
62
63	tcp = rte_pktmbuf_mtod_offset(m, struct tcp_hdr *, l2_len + l3_len);
64	return (tcp->data_off >> 4) * 4;
65}
66
67static inline void
68adjust_ipv4_pktlen(struct rte_mbuf *m, uint32_t l2_len)
69{
70	uint32_t plen, trim;
71	const struct ipv4_hdr *iph;
72
73	iph = rte_pktmbuf_mtod_offset(m, const struct ipv4_hdr *, l2_len);
74	plen = rte_be_to_cpu_16(iph->total_length) + l2_len;
75	if (plen < m->pkt_len) {
76		trim = m->pkt_len - plen;
77		rte_pktmbuf_trim(m, trim);
78	}
79}
80
81static inline void
82adjust_ipv6_pktlen(struct rte_mbuf *m, uint32_t l2_len)
83{
84	uint32_t plen, trim;
85	const struct ipv6_hdr *iph;
86
87	iph = rte_pktmbuf_mtod_offset(m, const struct ipv6_hdr *, l2_len);
88	plen = rte_be_to_cpu_16(iph->payload_len) + sizeof(*iph) + l2_len;
89	if (plen < m->pkt_len) {
90		trim = m->pkt_len - plen;
91		rte_pktmbuf_trim(m, trim);
92	}
93}
94
95static inline void
96tcp_stat_update(struct netbe_lcore *lc, const struct rte_mbuf *m,
97	uint32_t l2_len, uint32_t l3_len)
98{
99	const struct tcp_hdr *th;
100
101	th = rte_pktmbuf_mtod_offset(m, struct tcp_hdr *, l2_len + l3_len);
102	lc->tcp_stat.flags[th->tcp_flags]++;
103}
104
105static inline uint32_t
106get_ipv4_hdr_len(struct rte_mbuf *m, uint32_t l2, uint32_t proto, uint32_t frag)
107{
108	const struct ipv4_hdr *iph;
109	int32_t dlen, len;
110
111	dlen = rte_pktmbuf_data_len(m);
112	dlen -= l2;
113
114	iph = rte_pktmbuf_mtod_offset(m, const struct ipv4_hdr *, l2);
115	len = (iph->version_ihl & IPV4_HDR_IHL_MASK) * IPV4_IHL_MULTIPLIER;
116
117	if (frag != 0 && is_ipv4_frag(iph)) {
118		m->packet_type &= ~RTE_PTYPE_L4_MASK;
119		m->packet_type |= RTE_PTYPE_L4_FRAG;
120	}
121
122	if (len > dlen || (proto <= IPPROTO_MAX && iph->next_proto_id != proto))
123		m->packet_type = RTE_PTYPE_UNKNOWN;
124
125	return len;
126}
127
128static inline void
129fill_ipv4_hdr_len(struct rte_mbuf *m, uint32_t l2, uint32_t proto,
130	uint32_t frag, uint32_t l4_len)
131{
132	uint32_t len;
133
134	len = get_ipv4_hdr_len(m, l2, proto, frag);
135	fill_pkt_hdr_len(m, l2, len, l4_len);
136	adjust_ipv4_pktlen(m, l2);
137}
138
139static inline int
140ipv6x_hdr(uint32_t proto)
141{
142	return (proto == IPPROTO_HOPOPTS ||
143		proto == IPPROTO_ROUTING ||
144		proto == IPPROTO_FRAGMENT ||
145		proto == IPPROTO_AH ||
146		proto == IPPROTO_NONE ||
147		proto == IPPROTO_DSTOPTS);
148}
149
150static inline uint32_t
151get_ipv6x_hdr_len(struct rte_mbuf *m, uint32_t l2, uint32_t nproto,
152	uint32_t fproto)
153{
154	const struct ip6_ext *ipx;
155	int32_t dlen, len, ofs;
156
157	len = sizeof(struct ipv6_hdr);
158
159	dlen = rte_pktmbuf_data_len(m);
160	dlen -= l2;
161
162	ofs = l2 + len;
163	ipx = rte_pktmbuf_mtod_offset(m, const struct ip6_ext *, ofs);
164
165	while (ofs > 0 && len < dlen) {
166
167		switch (nproto) {
168		case IPPROTO_HOPOPTS:
169		case IPPROTO_ROUTING:
170		case IPPROTO_DSTOPTS:
171			ofs = (ipx->ip6e_len + 1) << 3;
172			break;
173		case IPPROTO_AH:
174			ofs = (ipx->ip6e_len + 2) << 2;
175			break;
176		case IPPROTO_FRAGMENT:
177			/*
178			 * tso_segsz is not used by RX, so use it as temporary
179			 * buffer to store the fragment offset.
180			 */
181			m->tso_segsz = ofs;
182			ofs = sizeof(struct ip6_frag);
183			m->packet_type &= ~RTE_PTYPE_L4_MASK;
184			m->packet_type |= RTE_PTYPE_L4_FRAG;
185			break;
186		default:
187			ofs = 0;
188		}
189
190		if (ofs > 0) {
191			nproto = ipx->ip6e_nxt;
192			len += ofs;
193			ipx += ofs / sizeof(*ipx);
194		}
195	}
196
197	/* unrecognized or invalid packet. */
198	if ((ofs == 0 && nproto != fproto) || len > dlen)
199		m->packet_type = RTE_PTYPE_UNKNOWN;
200
201	return len;
202}
203
204static inline uint32_t
205get_ipv6_hdr_len(struct rte_mbuf *m, uint32_t l2, uint32_t fproto)
206{
207	const struct ipv6_hdr *iph;
208
209	iph = rte_pktmbuf_mtod_offset(m, const struct ipv6_hdr *,
210		sizeof(struct ether_hdr));
211
212	if (iph->proto == fproto)
213		return sizeof(struct ipv6_hdr);
214	else if (ipv6x_hdr(iph->proto) != 0)
215		return get_ipv6x_hdr_len(m, l2, iph->proto, fproto);
216
217	m->packet_type = RTE_PTYPE_UNKNOWN;
218	return 0;
219}
220
221static inline void
222fill_ipv6_hdr_len(struct rte_mbuf *m, uint32_t l2, uint32_t fproto,
223	uint32_t l4_len)
224{
225	uint32_t len;
226
227	len = get_ipv6_hdr_len(m, l2, fproto);
228	fill_pkt_hdr_len(m, l2, len, l4_len);
229	adjust_ipv6_pktlen(m, l2);
230}
231
232static inline struct rte_mbuf *
233handle_arp(struct rte_mbuf *m, struct netbe_lcore *lc, uint8_t port,
234	uint32_t l2len)
235{
236	const struct arp_hdr *ahdr;
237	struct pkt_buf *abuf;
238
239	ahdr = rte_pktmbuf_mtod_offset(m, const struct arp_hdr *, l2len);
240
241	if (ahdr->arp_hrd != rte_be_to_cpu_16(ARP_HRD_ETHER) ||
242		ahdr->arp_pro != rte_be_to_cpu_16(ETHER_TYPE_IPv4) ||
243		ahdr->arp_op != rte_be_to_cpu_16(ARP_OP_REQUEST)) {
244
245		m->packet_type = RTE_PTYPE_UNKNOWN;
246		return m;
247	}
248
249	m->l2_len = l2len;
250	abuf = &lc->prtq[port].arp_buf;
251	if (abuf->num >= RTE_DIM(abuf->pkt))
252		return m;
253
254	abuf->pkt[abuf->num++] = m;
255
256	return NULL;
257}
258
259static inline struct rte_mbuf *
260fill_eth_tcp_arp_hdr_len(struct rte_mbuf *m, struct netbe_lcore *lc,
261	uint8_t port)
262{
263	uint32_t dlen, l2_len, l3_len, l4_len;
264	uint16_t etp;
265	const struct ether_hdr *eth;
266
267	dlen = rte_pktmbuf_data_len(m);
268
269	/* check that first segment is at least 54B long. */
270	if (dlen < sizeof(struct ether_hdr) + sizeof(struct ipv4_hdr) +
271			sizeof(struct tcp_hdr)) {
272		m->packet_type = RTE_PTYPE_UNKNOWN;
273		return m;
274	}
275
276	l2_len = sizeof(*eth);
277
278	eth = rte_pktmbuf_mtod(m, const struct ether_hdr *);
279	etp = eth->ether_type;
280	if (etp == rte_be_to_cpu_16(ETHER_TYPE_VLAN))
281		l2_len += sizeof(struct vlan_hdr);
282
283	if (etp == rte_be_to_cpu_16(ETHER_TYPE_ARP))
284		return handle_arp(m, lc, port, l2_len);
285
286	if (etp == rte_be_to_cpu_16(ETHER_TYPE_IPv4)) {
287		m->packet_type = RTE_PTYPE_L4_TCP |
288			RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
289			RTE_PTYPE_L2_ETHER;
290		l3_len = get_ipv4_hdr_len(m, l2_len, IPPROTO_TCP, 1);
291		l4_len = get_tcp_header_size(m, l2_len, l3_len);
292		fill_pkt_hdr_len(m, l2_len, l3_len, l4_len);
293		adjust_ipv4_pktlen(m, l2_len);
294	} else if (etp == rte_be_to_cpu_16(ETHER_TYPE_IPv6) &&
295			dlen >= l2_len + sizeof(struct ipv6_hdr) +
296			sizeof(struct tcp_hdr)) {
297		m->packet_type = RTE_PTYPE_L4_TCP |
298			RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
299			RTE_PTYPE_L2_ETHER;
300		l3_len = get_ipv6_hdr_len(m, l2_len, IPPROTO_TCP);
301		l4_len = get_tcp_header_size(m, l2_len, l3_len);
302		fill_pkt_hdr_len(m, l2_len, l3_len, l4_len);
303		adjust_ipv6_pktlen(m, l2_len);
304	} else
305		m->packet_type = RTE_PTYPE_UNKNOWN;
306
307	return m;
308}
309
310static inline void
311fill_eth_tcp_hdr_len(struct rte_mbuf *m)
312{
313	uint32_t dlen, l2_len, l3_len, l4_len;
314	uint16_t etp;
315	const struct ether_hdr *eth;
316
317	dlen = rte_pktmbuf_data_len(m);
318
319	/* check that first segment is at least 54B long. */
320	if (dlen < sizeof(struct ether_hdr) + sizeof(struct ipv4_hdr) +
321			sizeof(struct tcp_hdr)) {
322		m->packet_type = RTE_PTYPE_UNKNOWN;
323		return;
324	}
325
326	l2_len = sizeof(*eth);
327
328	eth = rte_pktmbuf_mtod(m, const struct ether_hdr *);
329	etp = eth->ether_type;
330	if (etp == rte_be_to_cpu_16(ETHER_TYPE_VLAN))
331		l2_len += sizeof(struct vlan_hdr);
332
333	if (etp == rte_be_to_cpu_16(ETHER_TYPE_IPv4)) {
334		m->packet_type = RTE_PTYPE_L4_TCP |
335			RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
336			RTE_PTYPE_L2_ETHER;
337		l3_len = get_ipv4_hdr_len(m, l2_len, IPPROTO_TCP, 1);
338		l4_len = get_tcp_header_size(m, l2_len, l3_len);
339		fill_pkt_hdr_len(m, l2_len, l3_len, l4_len);
340		adjust_ipv4_pktlen(m, l2_len);
341	} else if (etp == rte_be_to_cpu_16(ETHER_TYPE_IPv6) &&
342			dlen >= l2_len + sizeof(struct ipv6_hdr) +
343			sizeof(struct tcp_hdr)) {
344		m->packet_type = RTE_PTYPE_L4_TCP |
345			RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
346			RTE_PTYPE_L2_ETHER;
347		l3_len = get_ipv6_hdr_len(m, l2_len, IPPROTO_TCP);
348		l4_len = get_tcp_header_size(m, l2_len, l3_len);
349		fill_pkt_hdr_len(m, l2_len, l3_len, l4_len);
350		adjust_ipv6_pktlen(m, l2_len);
351	} else
352		m->packet_type = RTE_PTYPE_UNKNOWN;
353}
354
355static inline void
356fill_eth_udp_hdr_len(struct rte_mbuf *m)
357{
358	uint32_t dlen, l2_len;
359	uint16_t etp;
360	const struct ether_hdr *eth;
361
362	dlen = rte_pktmbuf_data_len(m);
363
364	/* check that first segment is at least 42B long. */
365	if (dlen < sizeof(struct ether_hdr) + sizeof(struct ipv4_hdr) +
366			sizeof(struct udp_hdr)) {
367		m->packet_type = RTE_PTYPE_UNKNOWN;
368		return;
369	}
370
371	l2_len = sizeof(*eth);
372
373	eth = rte_pktmbuf_mtod(m, const struct ether_hdr *);
374	etp = eth->ether_type;
375	if (etp == rte_be_to_cpu_16(ETHER_TYPE_VLAN))
376		l2_len += sizeof(struct vlan_hdr);
377
378	if (etp == rte_be_to_cpu_16(ETHER_TYPE_IPv4)) {
379		m->packet_type = RTE_PTYPE_L4_UDP |
380			RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
381			RTE_PTYPE_L2_ETHER;
382		fill_ipv4_hdr_len(m, l2_len, IPPROTO_UDP, 1,
383			sizeof(struct udp_hdr));
384	} else if (etp == rte_be_to_cpu_16(ETHER_TYPE_IPv6) &&
385			dlen >= l2_len + sizeof(struct ipv6_hdr) +
386			sizeof(struct udp_hdr)) {
387		m->packet_type = RTE_PTYPE_L4_UDP |
388			RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
389			RTE_PTYPE_L2_ETHER;
390		fill_ipv6_hdr_len(m, l2_len, IPPROTO_UDP,
391			sizeof(struct udp_hdr));
392	} else
393		m->packet_type = RTE_PTYPE_UNKNOWN;
394}
395
396static inline uint16_t
397ipv4x_cksum(const void *iph, size_t len)
398{
399	uint16_t cksum;
400
401	cksum = rte_raw_cksum(iph, len);
402	return (cksum == 0xffff) ? cksum : ~cksum;
403}
404
405static inline void
406fix_reassembled(struct rte_mbuf *m, int32_t hwcsum, uint32_t proto)
407{
408	struct ipv4_hdr *iph;
409
410	/* update packet type. */
411	m->packet_type &= ~RTE_PTYPE_L4_MASK;
412
413	if (proto == IPPROTO_TCP)
414		m->packet_type |= RTE_PTYPE_L4_TCP;
415	else
416		m->packet_type |= RTE_PTYPE_L4_UDP;
417
418	/* fix reassemble setting TX flags. */
419	m->ol_flags &= ~PKT_TX_IP_CKSUM;
420
421	/* fix l3_len after reassemble. */
422	if (RTE_ETH_IS_IPV6_HDR(m->packet_type))
423		m->l3_len = m->l3_len - sizeof(struct ipv6_extension_fragment);
424
425	/* recalculate ipv4 cksum after reassemble. */
426	else if (hwcsum == 0 && RTE_ETH_IS_IPV4_HDR(m->packet_type)) {
427		iph = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *, m->l2_len);
428		iph->hdr_checksum = ipv4x_cksum(iph, m->l3_len);
429	}
430}
431
432static struct rte_mbuf *
433reassemble(struct rte_mbuf *m, struct netbe_lcore *lc, uint64_t tms,
434	uint8_t port, uint32_t proto)
435{
436	uint32_t l3cs;
437	struct rte_ip_frag_tbl *tbl;
438	struct rte_ip_frag_death_row *dr;
439
440	tbl = lc->ftbl;
441	dr = &lc->death_row;
442	l3cs = lc->prtq[port].port.rx_offload & DEV_RX_OFFLOAD_IPV4_CKSUM;
443
444	if (RTE_ETH_IS_IPV4_HDR(m->packet_type)) {
445
446		struct ipv4_hdr *iph;
447
448		iph = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *, m->l2_len);
449
450		/* process this fragment. */
451		m = rte_ipv4_frag_reassemble_packet(tbl, dr, m, tms, iph);
452
453	} else if (RTE_ETH_IS_IPV6_HDR(m->packet_type)) {
454
455		struct ipv6_hdr *iph;
456		struct ipv6_extension_fragment *fhdr;
457
458		iph = rte_pktmbuf_mtod_offset(m, struct ipv6_hdr *, m->l2_len);
459
460		/*
461		 * we store fragment header offset in tso_segsz before
462		 * temporary, just to avoid another scan of ipv6 header.
463		 */
464		fhdr = rte_pktmbuf_mtod_offset(m,
465			struct ipv6_extension_fragment *, m->tso_segsz);
466		m->tso_segsz = 0;
467
468		/* process this fragment. */
469		m = rte_ipv6_frag_reassemble_packet(tbl, dr, m, tms, iph, fhdr);
470
471	} else {
472		rte_pktmbuf_free(m);
473		m = NULL;
474	}
475
476	/* got reassembled packet. */
477	if (m != NULL)
478		fix_reassembled(m, l3cs, proto);
479
480	return m;
481}
482
483/* exclude NULLs from the final list of packets. */
484static inline uint32_t
485compress_pkt_list(struct rte_mbuf *pkt[], uint32_t nb_pkt, uint32_t nb_zero)
486{
487	uint32_t i, j, k, l;
488
489	for (j = nb_pkt; nb_zero != 0 && j-- != 0; ) {
490
491		/* found a hole. */
492		if (pkt[j] == NULL) {
493
494			/* find how big is it. */
495			for (i = j; i-- != 0 && pkt[i] == NULL; )
496				;
497			/* fill the hole. */
498			for (k = j + 1, l = i + 1; k != nb_pkt; k++, l++)
499				pkt[l] = pkt[k];
500
501			nb_pkt -= j - i;
502			nb_zero -= j - i;
503			j = i + 1;
504		}
505	}
506
507	return nb_pkt;
508}
509
510/*
511 * if it is a fragment, try to reassemble it,
512 * if by some reason it can't be done, then
513 * set pkt[] entry to NULL.
514 */
515#define DO_REASSEMBLE(proto) \
516do { \
517	if ((pkt[j]->packet_type & RTE_PTYPE_L4_MASK) == \
518			RTE_PTYPE_L4_FRAG) { \
519		cts = (cts == 0) ? rte_rdtsc() : cts; \
520		pkt[j] = reassemble(pkt[j], lc, cts, port, (proto)); \
521		x += (pkt[j] == NULL); \
522	} \
523} while (0)
524
525/*
526 * HW can recognize L2/L3 with/without extensions/L4 (ixgbe/igb/fm10k)
527 */
528static uint16_t
529type0_tcp_rx_callback(__rte_unused uint8_t port, __rte_unused uint16_t queue,
530	struct rte_mbuf *pkt[], uint16_t nb_pkts,
531	__rte_unused uint16_t max_pkts, void *user_param)
532{
533	uint32_t j, tp;
534	struct netbe_lcore *lc;
535	uint32_t l4_len, l3_len, l2_len;
536	const struct ether_hdr *eth;
537
538	lc = user_param;
539	l2_len = sizeof(*eth);
540
541	RTE_SET_USED(lc);
542
543	for (j = 0; j != nb_pkts; j++) {
544
545		NETBE_PKT_DUMP(pkt[j]);
546
547		tp = pkt[j]->packet_type & (RTE_PTYPE_L4_MASK |
548			RTE_PTYPE_L3_MASK | RTE_PTYPE_L2_MASK);
549
550		switch (tp) {
551		/* non fragmented tcp packets. */
552		case (RTE_PTYPE_L4_TCP | RTE_PTYPE_L3_IPV4 |
553				RTE_PTYPE_L2_ETHER):
554			l4_len = get_tcp_header_size(pkt[j], l2_len,
555				sizeof(struct ipv4_hdr));
556			fill_pkt_hdr_len(pkt[j], l2_len,
557				sizeof(struct ipv4_hdr), l4_len);
558			adjust_ipv4_pktlen(pkt[j], l2_len);
559			break;
560		case (RTE_PTYPE_L4_TCP | RTE_PTYPE_L3_IPV6 |
561				RTE_PTYPE_L2_ETHER):
562			l4_len = get_tcp_header_size(pkt[j], l2_len,
563				sizeof(struct ipv6_hdr));
564			fill_pkt_hdr_len(pkt[j], l2_len,
565				sizeof(struct ipv6_hdr), l4_len);
566			adjust_ipv6_pktlen(pkt[j], l2_len);
567			break;
568		case (RTE_PTYPE_L4_TCP | RTE_PTYPE_L3_IPV4_EXT |
569				RTE_PTYPE_L2_ETHER):
570			l3_len = get_ipv4_hdr_len(pkt[j], l2_len,
571				IPPROTO_TCP, 0);
572			l4_len = get_tcp_header_size(pkt[j], l2_len, l3_len);
573			fill_pkt_hdr_len(pkt[j], l2_len, l3_len, l4_len);
574			adjust_ipv4_pktlen(pkt[j], l2_len);
575			break;
576		case (RTE_PTYPE_L4_TCP | RTE_PTYPE_L3_IPV6_EXT |
577				RTE_PTYPE_L2_ETHER):
578			l3_len = get_ipv6_hdr_len(pkt[j], l2_len, IPPROTO_TCP);
579			l4_len = get_tcp_header_size(pkt[j], l2_len, l3_len);
580			fill_pkt_hdr_len(pkt[j], l2_len, l3_len, l4_len);
581			adjust_ipv6_pktlen(pkt[j], l2_len);
582			break;
583		default:
584			/* treat packet types as invalid. */
585			pkt[j]->packet_type = RTE_PTYPE_UNKNOWN;
586			break;
587		}
588	}
589
590	return nb_pkts;
591}
592
593/*
594 * HW can recognize L2/L3 with/without extensions/L4 (ixgbe/igb/fm10k)
595 */
596static uint16_t
597type0_udp_rx_callback(uint8_t port, __rte_unused uint16_t queue,
598	struct rte_mbuf *pkt[], uint16_t nb_pkts,
599	__rte_unused uint16_t max_pkts, void *user_param)
600{
601	uint32_t j, tp, x;
602	uint64_t cts;
603	struct netbe_lcore *lc;
604	uint32_t l2_len;
605	const struct ether_hdr *eth;
606
607	lc = user_param;
608	cts = 0;
609	l2_len = sizeof(*eth);
610
611	x = 0;
612	for (j = 0; j != nb_pkts; j++) {
613
614		NETBE_PKT_DUMP(pkt[j]);
615
616		tp = pkt[j]->packet_type & (RTE_PTYPE_L4_MASK |
617			RTE_PTYPE_L3_MASK | RTE_PTYPE_L2_MASK);
618
619		switch (tp) {
620		/* non fragmented udp packets. */
621		case (RTE_PTYPE_L4_UDP | RTE_PTYPE_L3_IPV4 |
622				RTE_PTYPE_L2_ETHER):
623			fill_pkt_hdr_len(pkt[j], l2_len,
624				sizeof(struct ipv4_hdr),
625				sizeof(struct udp_hdr));
626			adjust_ipv4_pktlen(pkt[j], l2_len);
627			break;
628		case (RTE_PTYPE_L4_UDP | RTE_PTYPE_L3_IPV6 |
629				RTE_PTYPE_L2_ETHER):
630			fill_pkt_hdr_len(pkt[j], l2_len,
631				sizeof(struct ipv6_hdr),
632				sizeof(struct udp_hdr));
633			adjust_ipv6_pktlen(pkt[j], l2_len);
634			break;
635		case (RTE_PTYPE_L4_UDP | RTE_PTYPE_L3_IPV4_EXT |
636				RTE_PTYPE_L2_ETHER):
637			fill_ipv4_hdr_len(pkt[j], l2_len,
638				UINT32_MAX, 0, sizeof(struct udp_hdr));
639			break;
640		case (RTE_PTYPE_L4_UDP | RTE_PTYPE_L3_IPV6_EXT |
641				RTE_PTYPE_L2_ETHER):
642			fill_ipv6_hdr_len(pkt[j], l2_len,
643				IPPROTO_UDP, sizeof(struct udp_hdr));
644			break;
645		/* possibly fragmented udp packets. */
646		case (RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L2_ETHER):
647		case (RTE_PTYPE_L3_IPV4_EXT | RTE_PTYPE_L2_ETHER):
648			fill_ipv4_hdr_len(pkt[j], l2_len,
649				IPPROTO_UDP, 1, sizeof(struct udp_hdr));
650			break;
651		case (RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L2_ETHER):
652		case (RTE_PTYPE_L3_IPV6_EXT | RTE_PTYPE_L2_ETHER):
653			fill_ipv6_hdr_len(pkt[j], l2_len,
654				IPPROTO_UDP, sizeof(struct udp_hdr));
655			break;
656		default:
657			/* treat packet types as invalid. */
658			pkt[j]->packet_type = RTE_PTYPE_UNKNOWN;
659			break;
660		}
661
662		DO_REASSEMBLE(IPPROTO_UDP);
663	}
664
665	/* reassemble was invoked, cleanup its death-row. */
666	if (cts != 0)
667		rte_ip_frag_free_death_row(&lc->death_row, 0);
668
669	if (x == 0)
670		return nb_pkts;
671
672	NETBE_TRACE("%s(port=%u, queue=%u, nb_pkts=%u): "
673		"%u non-reassembled fragments;\n",
674		__func__, port, queue, nb_pkts, x);
675
676	return compress_pkt_list(pkt, nb_pkts, x);
677}
678
679/*
680 * HW can recognize L2/L3/L4 and fragments (i40e).
681 */
682static uint16_t
683type1_tcp_rx_callback(__rte_unused uint8_t port, __rte_unused uint16_t queue,
684	struct rte_mbuf *pkt[], uint16_t nb_pkts,
685	__rte_unused uint16_t max_pkts, void *user_param)
686{
687	uint32_t j, tp;
688	struct netbe_lcore *lc;
689	uint32_t l4_len, l3_len, l2_len;
690	const struct ether_hdr *eth;
691
692	lc = user_param;
693	l2_len = sizeof(*eth);
694
695	RTE_SET_USED(lc);
696
697	for (j = 0; j != nb_pkts; j++) {
698
699		NETBE_PKT_DUMP(pkt[j]);
700
701		tp = pkt[j]->packet_type & (RTE_PTYPE_L4_MASK |
702			RTE_PTYPE_L3_MASK | RTE_PTYPE_L2_MASK);
703
704		switch (tp) {
705		case (RTE_PTYPE_L4_TCP | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
706				RTE_PTYPE_L2_ETHER):
707			l3_len = get_ipv4_hdr_len(pkt[j], l2_len,
708				IPPROTO_TCP, 0);
709			l4_len = get_tcp_header_size(pkt[j], l2_len, l3_len);
710			fill_pkt_hdr_len(pkt[j], l2_len, l3_len, l4_len);
711			adjust_ipv4_pktlen(pkt[j], l2_len);
712			tcp_stat_update(lc, pkt[j], l2_len, l3_len);
713			break;
714		case (RTE_PTYPE_L4_TCP | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
715				RTE_PTYPE_L2_ETHER):
716			l3_len = get_ipv6_hdr_len(pkt[j], l2_len, IPPROTO_TCP);
717			l4_len = get_tcp_header_size(pkt[j], l2_len, l3_len);
718			fill_pkt_hdr_len(pkt[j], l2_len, l3_len, l4_len);
719			adjust_ipv6_pktlen(pkt[j], l2_len);
720			tcp_stat_update(lc, pkt[j], l2_len, l3_len);
721			break;
722		default:
723			/* treat packet types as invalid. */
724			pkt[j]->packet_type = RTE_PTYPE_UNKNOWN;
725			break;
726		}
727
728	}
729
730	return nb_pkts;
731}
732
733/*
734 * HW can recognize L2/L3/L4 and fragments (i40e).
735 */
736static uint16_t
737type1_udp_rx_callback(uint8_t port, __rte_unused uint16_t queue,
738	struct rte_mbuf *pkt[], uint16_t nb_pkts,
739	__rte_unused uint16_t max_pkts, void *user_param)
740{
741	uint32_t j, tp, x;
742	uint64_t cts;
743	struct netbe_lcore *lc;
744	uint32_t l2_len;
745	const struct ether_hdr *eth;
746
747	lc = user_param;
748	cts = 0;
749	l2_len = sizeof(*eth);
750
751	x = 0;
752	for (j = 0; j != nb_pkts; j++) {
753
754		NETBE_PKT_DUMP(pkt[j]);
755
756		tp = pkt[j]->packet_type & (RTE_PTYPE_L4_MASK |
757			RTE_PTYPE_L3_MASK | RTE_PTYPE_L2_MASK);
758
759		switch (tp) {
760		case (RTE_PTYPE_L4_UDP | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
761				RTE_PTYPE_L2_ETHER):
762			fill_ipv4_hdr_len(pkt[j], l2_len,
763				UINT32_MAX, 0, sizeof(struct udp_hdr));
764			break;
765		case (RTE_PTYPE_L4_UDP | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
766				RTE_PTYPE_L2_ETHER):
767			fill_ipv6_hdr_len(pkt[j], l2_len,
768				IPPROTO_UDP, sizeof(struct udp_hdr));
769			break;
770		case (RTE_PTYPE_L4_FRAG | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
771				RTE_PTYPE_L2_ETHER):
772			fill_ipv4_hdr_len(pkt[j], l2_len,
773				IPPROTO_UDP, 0, sizeof(struct udp_hdr));
774			break;
775		case (RTE_PTYPE_L4_FRAG | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
776				RTE_PTYPE_L2_ETHER):
777			fill_ipv6_hdr_len(pkt[j], l2_len,
778				IPPROTO_UDP, sizeof(struct udp_hdr));
779			break;
780		default:
781			/* treat packet types as invalid. */
782			pkt[j]->packet_type = RTE_PTYPE_UNKNOWN;
783			break;
784		}
785
786		DO_REASSEMBLE(IPPROTO_UDP);
787	}
788
789	/* reassemble was invoked, cleanup its death-row. */
790	if (cts != 0)
791		rte_ip_frag_free_death_row(&lc->death_row, 0);
792
793	if (x == 0)
794		return nb_pkts;
795
796	NETBE_TRACE("%s(port=%u, queue=%u, nb_pkts=%u): "
797		"%u non-reassembled fragments;\n",
798		__func__, port, queue, nb_pkts, x);
799
800	return compress_pkt_list(pkt, nb_pkts, x);
801}
802
803/*
804 * generic, assumes HW doesn't recognize any packet type.
805 */
806static uint16_t
807typen_tcp_arp_rx_callback(uint8_t port, uint16_t queue, struct rte_mbuf *pkt[],
808	uint16_t nb_pkts, uint16_t max_pkts, void *user_param)
809{
810	uint32_t j, x;
811	struct netbe_lcore *lc;
812
813	lc = user_param;
814
815	RTE_SET_USED(queue);
816	RTE_SET_USED(max_pkts);
817
818	x = 0;
819	for (j = 0; j != nb_pkts; j++) {
820
821		NETBE_PKT_DUMP(pkt[j]);
822		pkt[j] = fill_eth_tcp_arp_hdr_len(pkt[j], lc, port);
823		x += (pkt[j] == NULL);
824	}
825
826	if (x == 0)
827		return nb_pkts;
828
829	return compress_pkt_list(pkt, nb_pkts, x);
830}
831
832static uint16_t
833typen_tcp_rx_callback(__rte_unused uint8_t port, __rte_unused uint16_t queue,
834	struct rte_mbuf *pkt[], uint16_t nb_pkts,
835	__rte_unused uint16_t max_pkts, void *user_param)
836{
837	uint32_t j;
838	struct netbe_lcore *lc;
839
840	lc = user_param;
841
842	RTE_SET_USED(lc);
843
844	for (j = 0; j != nb_pkts; j++) {
845
846		NETBE_PKT_DUMP(pkt[j]);
847		fill_eth_tcp_hdr_len(pkt[j]);
848	}
849
850	return nb_pkts;
851}
852
853static uint16_t
854typen_udp_rx_callback(uint8_t port, __rte_unused uint16_t queue,
855	struct rte_mbuf *pkt[], uint16_t nb_pkts,
856	__rte_unused uint16_t max_pkts, void *user_param)
857{
858	uint32_t j, x;
859	uint64_t cts;
860	struct netbe_lcore *lc;
861
862	lc = user_param;
863	cts = 0;
864
865	x = 0;
866	for (j = 0; j != nb_pkts; j++) {
867
868		NETBE_PKT_DUMP(pkt[j]);
869		fill_eth_udp_hdr_len(pkt[j]);
870
871		DO_REASSEMBLE(IPPROTO_UDP);
872	}
873
874	/* reassemble was invoked, cleanup its death-row. */
875	if (cts != 0)
876		rte_ip_frag_free_death_row(&lc->death_row, 0);
877
878	if (x == 0)
879		return nb_pkts;
880
881	NETBE_TRACE("%s(port=%u, queue=%u, nb_pkts=%u): "
882		"%u non-reassembled fragments;\n",
883		__func__, port, queue, nb_pkts, x);
884
885	return compress_pkt_list(pkt, nb_pkts, x);
886}
887
888static uint32_t
889get_ptypes(const struct netbe_port *uprt)
890{
891	uint32_t smask;
892	int32_t i, rc;
893	const uint32_t pmask = RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK |
894		RTE_PTYPE_L4_MASK;
895
896	smask = 0;
897	rc = rte_eth_dev_get_supported_ptypes(uprt->id, pmask, NULL, 0);
898	if (rc < 0) {
899		RTE_LOG(ERR, USER1,
900			"%s(port=%u) failed to get supported ptypes;\n",
901			__func__, uprt->id);
902		return smask;
903	}
904
905	uint32_t ptype[rc];
906	rc = rte_eth_dev_get_supported_ptypes(uprt->id, pmask, ptype, rc);
907
908	for (i = 0; i != rc; i++) {
909		switch (ptype[i]) {
910		case RTE_PTYPE_L2_ETHER:
911			smask |= ETHER_PTYPE;
912			break;
913		case RTE_PTYPE_L3_IPV4:
914		case RTE_PTYPE_L3_IPV4_EXT_UNKNOWN:
915			smask |= IPV4_PTYPE;
916			break;
917		case RTE_PTYPE_L3_IPV4_EXT:
918			smask |= IPV4_EXT_PTYPE;
919			break;
920		case RTE_PTYPE_L3_IPV6:
921		case RTE_PTYPE_L3_IPV6_EXT_UNKNOWN:
922			smask |= IPV6_PTYPE;
923			break;
924		case RTE_PTYPE_L3_IPV6_EXT:
925			smask |= IPV6_EXT_PTYPE;
926			break;
927		case RTE_PTYPE_L4_TCP:
928			smask |= TCP_PTYPE;
929			break;
930		case RTE_PTYPE_L4_UDP:
931			smask |= UDP_PTYPE;
932			break;
933		}
934	}
935
936	return smask;
937}
938
939int
940setup_rx_cb(const struct netbe_port *uprt, struct netbe_lcore *lc,
941	uint16_t qid, uint32_t arp)
942{
943	int32_t rc;
944	uint32_t i, n, smask;
945	void *cb;
946	const struct ptype2cb *ptype2cb;
947
948	static const struct ptype2cb tcp_ptype2cb[] = {
949		{
950			.mask = ETHER_PTYPE | IPV4_PTYPE | IPV4_EXT_PTYPE |
951				IPV6_PTYPE | IPV6_EXT_PTYPE | TCP_PTYPE,
952			.name = "HW l2/l3x/l4-tcp ptype",
953			.fn = type0_tcp_rx_callback,
954		},
955		{
956			.mask = ETHER_PTYPE | IPV4_PTYPE | IPV6_PTYPE |
957				TCP_PTYPE,
958			.name = "HW l2/l3/l4-tcp ptype",
959			.fn = type1_tcp_rx_callback,
960		},
961		{
962			.mask = 0,
963			.name = "tcp no HW ptype",
964			.fn = typen_tcp_rx_callback,
965		},
966	};
967
968	static const struct ptype2cb tcp_arp_ptype2cb[] = {
969		{
970			.mask = 0,
971			.name = "tcp with arp no HW ptype",
972			.fn = typen_tcp_arp_rx_callback,
973		},
974	};
975
976	static const struct ptype2cb udp_ptype2cb[] = {
977		{
978			.mask = ETHER_PTYPE | IPV4_PTYPE | IPV4_EXT_PTYPE |
979				IPV6_PTYPE | IPV6_EXT_PTYPE | UDP_PTYPE,
980			.name = "HW l2/l3x/l4-udp ptype",
981			.fn = type0_udp_rx_callback,
982		},
983		{
984			.mask = ETHER_PTYPE | IPV4_PTYPE | IPV6_PTYPE |
985				UDP_PTYPE,
986			.name = "HW l2/l3/l4-udp ptype",
987			.fn = type1_udp_rx_callback,
988		},
989		{
990			.mask = 0,
991			.name = "udp no HW ptype",
992			.fn = typen_udp_rx_callback,
993		},
994	};
995
996	smask = get_ptypes(uprt);
997
998	if (lc->proto == TLE_PROTO_TCP) {
999		if (arp != 0) {
1000			ptype2cb = tcp_arp_ptype2cb;
1001			n = RTE_DIM(tcp_arp_ptype2cb);
1002		} else {
1003			ptype2cb = tcp_ptype2cb;
1004			n = RTE_DIM(tcp_ptype2cb);
1005		}
1006	} else if (lc->proto == TLE_PROTO_UDP) {
1007		ptype2cb = udp_ptype2cb;
1008		n = RTE_DIM(udp_ptype2cb);
1009	} else {
1010		RTE_LOG(ERR, USER1,
1011			"%s(lc=%u) unsupported proto: %u\n",
1012			__func__, lc->id, lc->proto);
1013		return -EINVAL;
1014	}
1015
1016	for (i = 0; i != n; i++) {
1017		if ((smask & ptype2cb[i].mask) == ptype2cb[i].mask) {
1018			cb = rte_eth_add_rx_callback(uprt->id, qid,
1019				ptype2cb[i].fn, lc);
1020			rc = -rte_errno;
1021			RTE_LOG(ERR, USER1,
1022				"%s(port=%u), setup RX callback \"%s\" "
1023				"returns %p;\n",
1024				__func__, uprt->id,  ptype2cb[i].name, cb);
1025				return ((cb == NULL) ? rc : 0);
1026		}
1027	}
1028
1029	/* no proper callback found. */
1030	RTE_LOG(ERR, USER1,
1031		"%s(port=%u) failed to find an appropriate callback;\n",
1032		__func__, uprt->id);
1033	return -ENOENT;
1034}
1035
1036