i40e_rxtx.c revision c300e355
1/*-
2 *   BSD LICENSE
3 *
4 *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
5 *   All rights reserved.
6 *
7 *   Redistribution and use in source and binary forms, with or without
8 *   modification, are permitted provided that the following conditions
9 *   are met:
10 *
11 *     * Redistributions of source code must retain the above copyright
12 *       notice, this list of conditions and the following disclaimer.
13 *     * Redistributions in binary form must reproduce the above copyright
14 *       notice, this list of conditions and the following disclaimer in
15 *       the documentation and/or other materials provided with the
16 *       distribution.
17 *     * Neither the name of Intel Corporation nor the names of its
18 *       contributors may be used to endorse or promote products derived
19 *       from this software without specific prior written permission.
20 *
21 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34#include <stdio.h>
35#include <stdlib.h>
36#include <string.h>
37#include <errno.h>
38#include <stdint.h>
39#include <stdarg.h>
40#include <unistd.h>
41#include <inttypes.h>
42#include <sys/queue.h>
43
44#include <rte_string_fns.h>
45#include <rte_memzone.h>
46#include <rte_mbuf.h>
47#include <rte_malloc.h>
48#include <rte_ether.h>
49#include <rte_ethdev.h>
50#include <rte_tcp.h>
51#include <rte_sctp.h>
52#include <rte_udp.h>
53
54#include "i40e_logs.h"
55#include "base/i40e_prototype.h"
56#include "base/i40e_type.h"
57#include "i40e_ethdev.h"
58#include "i40e_rxtx.h"
59
60#define DEFAULT_TX_RS_THRESH   32
61#define DEFAULT_TX_FREE_THRESH 32
62#define I40E_MAX_PKT_TYPE      256
63
64#define I40E_TX_MAX_BURST  32
65
66#define I40E_DMA_MEM_ALIGN 4096
67
68/* Base address of the HW descriptor ring should be 128B aligned. */
69#define I40E_RING_BASE_ALIGN	128
70
71#define I40E_SIMPLE_FLAGS ((uint32_t)ETH_TXQ_FLAGS_NOMULTSEGS | \
72					ETH_TXQ_FLAGS_NOOFFLOADS)
73
74#define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS)
75
76#define I40E_TX_CKSUM_OFFLOAD_MASK (		 \
77		PKT_TX_IP_CKSUM |		 \
78		PKT_TX_L4_MASK |		 \
79		PKT_TX_TCP_SEG |		 \
80		PKT_TX_OUTER_IP_CKSUM)
81
82static uint16_t i40e_xmit_pkts_simple(void *tx_queue,
83				      struct rte_mbuf **tx_pkts,
84				      uint16_t nb_pkts);
85
86static inline void
87i40e_rxd_to_vlan_tci(struct rte_mbuf *mb, volatile union i40e_rx_desc *rxdp)
88{
89	if (rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len) &
90		(1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)) {
91		mb->ol_flags |= PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED;
92		mb->vlan_tci =
93			rte_le_to_cpu_16(rxdp->wb.qword0.lo_dword.l2tag1);
94		PMD_RX_LOG(DEBUG, "Descriptor l2tag1: %u",
95			   rte_le_to_cpu_16(rxdp->wb.qword0.lo_dword.l2tag1));
96	} else {
97		mb->vlan_tci = 0;
98	}
99#ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC
100	if (rte_le_to_cpu_16(rxdp->wb.qword2.ext_status) &
101		(1 << I40E_RX_DESC_EXT_STATUS_L2TAG2P_SHIFT)) {
102		mb->ol_flags |= PKT_RX_QINQ_STRIPPED;
103		mb->vlan_tci_outer = mb->vlan_tci;
104		mb->vlan_tci = rte_le_to_cpu_16(rxdp->wb.qword2.l2tag2_2);
105		PMD_RX_LOG(DEBUG, "Descriptor l2tag2_1: %u, l2tag2_2: %u",
106			   rte_le_to_cpu_16(rxdp->wb.qword2.l2tag2_1),
107			   rte_le_to_cpu_16(rxdp->wb.qword2.l2tag2_2));
108	} else {
109		mb->vlan_tci_outer = 0;
110	}
111#endif
112	PMD_RX_LOG(DEBUG, "Mbuf vlan_tci: %u, vlan_tci_outer: %u",
113		   mb->vlan_tci, mb->vlan_tci_outer);
114}
115
116/* Translate the rx descriptor status to pkt flags */
117static inline uint64_t
118i40e_rxd_status_to_pkt_flags(uint64_t qword)
119{
120	uint64_t flags;
121
122	/* Check if RSS_HASH */
123	flags = (((qword >> I40E_RX_DESC_STATUS_FLTSTAT_SHIFT) &
124					I40E_RX_DESC_FLTSTAT_RSS_HASH) ==
125			I40E_RX_DESC_FLTSTAT_RSS_HASH) ? PKT_RX_RSS_HASH : 0;
126
127	/* Check if FDIR Match */
128	flags |= (qword & (1 << I40E_RX_DESC_STATUS_FLM_SHIFT) ?
129							PKT_RX_FDIR : 0);
130
131	return flags;
132}
133
134static inline uint64_t
135i40e_rxd_error_to_pkt_flags(uint64_t qword)
136{
137	uint64_t flags = 0;
138	uint64_t error_bits = (qword >> I40E_RXD_QW1_ERROR_SHIFT);
139
140#define I40E_RX_ERR_BITS 0x3f
141	if (likely((error_bits & I40E_RX_ERR_BITS) == 0))
142		return flags;
143	if (unlikely(error_bits & (1 << I40E_RX_DESC_ERROR_IPE_SHIFT)))
144		flags |= PKT_RX_IP_CKSUM_BAD;
145	if (unlikely(error_bits & (1 << I40E_RX_DESC_ERROR_L4E_SHIFT)))
146		flags |= PKT_RX_L4_CKSUM_BAD;
147	if (unlikely(error_bits & (1 << I40E_RX_DESC_ERROR_EIPE_SHIFT)))
148		flags |= PKT_RX_EIP_CKSUM_BAD;
149
150	return flags;
151}
152
153/* Function to check and set the ieee1588 timesync index and get the
154 * appropriate flags.
155 */
156#ifdef RTE_LIBRTE_IEEE1588
157static inline uint64_t
158i40e_get_iee15888_flags(struct rte_mbuf *mb, uint64_t qword)
159{
160	uint64_t pkt_flags = 0;
161	uint16_t tsyn = (qword & (I40E_RXD_QW1_STATUS_TSYNVALID_MASK
162				  | I40E_RXD_QW1_STATUS_TSYNINDX_MASK))
163				    >> I40E_RX_DESC_STATUS_TSYNINDX_SHIFT;
164
165	if ((mb->packet_type & RTE_PTYPE_L2_MASK)
166			== RTE_PTYPE_L2_ETHER_TIMESYNC)
167		pkt_flags = PKT_RX_IEEE1588_PTP;
168	if (tsyn & 0x04) {
169		pkt_flags |= PKT_RX_IEEE1588_TMST;
170		mb->timesync = tsyn & 0x03;
171	}
172
173	return pkt_flags;
174}
175#endif
176
177/* For each value it means, datasheet of hardware can tell more details
178 *
179 * @note: fix i40e_dev_supported_ptypes_get() if any change here.
180 */
181static inline uint32_t
182i40e_rxd_pkt_type_mapping(uint8_t ptype)
183{
184	static const uint32_t type_table[UINT8_MAX + 1] __rte_cache_aligned = {
185		/* L2 types */
186		/* [0] reserved */
187		[1] = RTE_PTYPE_L2_ETHER,
188		[2] = RTE_PTYPE_L2_ETHER_TIMESYNC,
189		/* [3] - [5] reserved */
190		[6] = RTE_PTYPE_L2_ETHER_LLDP,
191		/* [7] - [10] reserved */
192		[11] = RTE_PTYPE_L2_ETHER_ARP,
193		/* [12] - [21] reserved */
194
195		/* Non tunneled IPv4 */
196		[22] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
197			RTE_PTYPE_L4_FRAG,
198		[23] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
199			RTE_PTYPE_L4_NONFRAG,
200		[24] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
201			RTE_PTYPE_L4_UDP,
202		/* [25] reserved */
203		[26] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
204			RTE_PTYPE_L4_TCP,
205		[27] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
206			RTE_PTYPE_L4_SCTP,
207		[28] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
208			RTE_PTYPE_L4_ICMP,
209
210		/* IPv4 --> IPv4 */
211		[29] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
212			RTE_PTYPE_TUNNEL_IP |
213			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
214			RTE_PTYPE_INNER_L4_FRAG,
215		[30] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
216			RTE_PTYPE_TUNNEL_IP |
217			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
218			RTE_PTYPE_INNER_L4_NONFRAG,
219		[31] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
220			RTE_PTYPE_TUNNEL_IP |
221			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
222			RTE_PTYPE_INNER_L4_UDP,
223		/* [32] reserved */
224		[33] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
225			RTE_PTYPE_TUNNEL_IP |
226			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
227			RTE_PTYPE_INNER_L4_TCP,
228		[34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
229			RTE_PTYPE_TUNNEL_IP |
230			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
231			RTE_PTYPE_INNER_L4_SCTP,
232		[35] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
233			RTE_PTYPE_TUNNEL_IP |
234			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
235			RTE_PTYPE_INNER_L4_ICMP,
236
237		/* IPv4 --> IPv6 */
238		[36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
239			RTE_PTYPE_TUNNEL_IP |
240			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
241			RTE_PTYPE_INNER_L4_FRAG,
242		[37] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
243			RTE_PTYPE_TUNNEL_IP |
244			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
245			RTE_PTYPE_INNER_L4_NONFRAG,
246		[38] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
247			RTE_PTYPE_TUNNEL_IP |
248			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
249			RTE_PTYPE_INNER_L4_UDP,
250		/* [39] reserved */
251		[40] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
252			RTE_PTYPE_TUNNEL_IP |
253			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
254			RTE_PTYPE_INNER_L4_TCP,
255		[41] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
256			RTE_PTYPE_TUNNEL_IP |
257			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
258			RTE_PTYPE_INNER_L4_SCTP,
259		[42] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
260			RTE_PTYPE_TUNNEL_IP |
261			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
262			RTE_PTYPE_INNER_L4_ICMP,
263
264		/* IPv4 --> GRE/Teredo/VXLAN */
265		[43] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
266			RTE_PTYPE_TUNNEL_GRENAT,
267
268		/* IPv4 --> GRE/Teredo/VXLAN --> IPv4 */
269		[44] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
270			RTE_PTYPE_TUNNEL_GRENAT |
271			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
272			RTE_PTYPE_INNER_L4_FRAG,
273		[45] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
274			RTE_PTYPE_TUNNEL_GRENAT |
275			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
276			RTE_PTYPE_INNER_L4_NONFRAG,
277		[46] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
278			RTE_PTYPE_TUNNEL_GRENAT |
279			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
280			RTE_PTYPE_INNER_L4_UDP,
281		/* [47] reserved */
282		[48] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
283			RTE_PTYPE_TUNNEL_GRENAT |
284			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
285			RTE_PTYPE_INNER_L4_TCP,
286		[49] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
287			RTE_PTYPE_TUNNEL_GRENAT |
288			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
289			RTE_PTYPE_INNER_L4_SCTP,
290		[50] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
291			RTE_PTYPE_TUNNEL_GRENAT |
292			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
293			RTE_PTYPE_INNER_L4_ICMP,
294
295		/* IPv4 --> GRE/Teredo/VXLAN --> IPv6 */
296		[51] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
297			RTE_PTYPE_TUNNEL_GRENAT |
298			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
299			RTE_PTYPE_INNER_L4_FRAG,
300		[52] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
301			RTE_PTYPE_TUNNEL_GRENAT |
302			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
303			RTE_PTYPE_INNER_L4_NONFRAG,
304		[53] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
305			RTE_PTYPE_TUNNEL_GRENAT |
306			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
307			RTE_PTYPE_INNER_L4_UDP,
308		/* [54] reserved */
309		[55] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
310			RTE_PTYPE_TUNNEL_GRENAT |
311			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
312			RTE_PTYPE_INNER_L4_TCP,
313		[56] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
314			RTE_PTYPE_TUNNEL_GRENAT |
315			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
316			RTE_PTYPE_INNER_L4_SCTP,
317		[57] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
318			RTE_PTYPE_TUNNEL_GRENAT |
319			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
320			RTE_PTYPE_INNER_L4_ICMP,
321
322		/* IPv4 --> GRE/Teredo/VXLAN --> MAC */
323		[58] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
324			RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER,
325
326		/* IPv4 --> GRE/Teredo/VXLAN --> MAC --> IPv4 */
327		[59] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
328			RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
329			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
330			RTE_PTYPE_INNER_L4_FRAG,
331		[60] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
332			RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
333			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
334			RTE_PTYPE_INNER_L4_NONFRAG,
335		[61] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
336			RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
337			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
338			RTE_PTYPE_INNER_L4_UDP,
339		/* [62] reserved */
340		[63] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
341			RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
342			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
343			RTE_PTYPE_INNER_L4_TCP,
344		[64] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
345			RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
346			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
347			RTE_PTYPE_INNER_L4_SCTP,
348		[65] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
349			RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
350			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
351			RTE_PTYPE_INNER_L4_ICMP,
352
353		/* IPv4 --> GRE/Teredo/VXLAN --> MAC --> IPv6 */
354		[66] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
355			RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
356			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
357			RTE_PTYPE_INNER_L4_FRAG,
358		[67] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
359			RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
360			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
361			RTE_PTYPE_INNER_L4_NONFRAG,
362		[68] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
363			RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
364			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
365			RTE_PTYPE_INNER_L4_UDP,
366		/* [69] reserved */
367		[70] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
368			RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
369			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
370			RTE_PTYPE_INNER_L4_TCP,
371		[71] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
372			RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
373			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
374			RTE_PTYPE_INNER_L4_SCTP,
375		[72] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
376			RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
377			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
378			RTE_PTYPE_INNER_L4_ICMP,
379
380		/* IPv4 --> GRE/Teredo/VXLAN --> MAC/VLAN */
381		[73] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
382			RTE_PTYPE_TUNNEL_GRENAT |
383			RTE_PTYPE_INNER_L2_ETHER_VLAN,
384
385		/* IPv4 --> GRE/Teredo/VXLAN --> MAC/VLAN --> IPv4 */
386		[74] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
387			RTE_PTYPE_TUNNEL_GRENAT |
388			RTE_PTYPE_INNER_L2_ETHER_VLAN |
389			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
390			RTE_PTYPE_INNER_L4_FRAG,
391		[75] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
392			RTE_PTYPE_TUNNEL_GRENAT |
393			RTE_PTYPE_INNER_L2_ETHER_VLAN |
394			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
395			RTE_PTYPE_INNER_L4_NONFRAG,
396		[76] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
397			RTE_PTYPE_TUNNEL_GRENAT |
398			RTE_PTYPE_INNER_L2_ETHER_VLAN |
399			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
400			RTE_PTYPE_INNER_L4_UDP,
401		/* [77] reserved */
402		[78] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
403			RTE_PTYPE_TUNNEL_GRENAT |
404			RTE_PTYPE_INNER_L2_ETHER_VLAN |
405			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
406			RTE_PTYPE_INNER_L4_TCP,
407		[79] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
408			RTE_PTYPE_TUNNEL_GRENAT |
409			RTE_PTYPE_INNER_L2_ETHER_VLAN |
410			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
411			RTE_PTYPE_INNER_L4_SCTP,
412		[80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
413			RTE_PTYPE_TUNNEL_GRENAT |
414			RTE_PTYPE_INNER_L2_ETHER_VLAN |
415			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
416			RTE_PTYPE_INNER_L4_ICMP,
417
418		/* IPv4 --> GRE/Teredo/VXLAN --> MAC/VLAN --> IPv6 */
419		[81] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
420			RTE_PTYPE_TUNNEL_GRENAT |
421			RTE_PTYPE_INNER_L2_ETHER_VLAN |
422			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
423			RTE_PTYPE_INNER_L4_FRAG,
424		[82] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
425			RTE_PTYPE_TUNNEL_GRENAT |
426			RTE_PTYPE_INNER_L2_ETHER_VLAN |
427			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
428			RTE_PTYPE_INNER_L4_NONFRAG,
429		[83] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
430			RTE_PTYPE_TUNNEL_GRENAT |
431			RTE_PTYPE_INNER_L2_ETHER_VLAN |
432			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
433			RTE_PTYPE_INNER_L4_UDP,
434		/* [84] reserved */
435		[85] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
436			RTE_PTYPE_TUNNEL_GRENAT |
437			RTE_PTYPE_INNER_L2_ETHER_VLAN |
438			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
439			RTE_PTYPE_INNER_L4_TCP,
440		[86] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
441			RTE_PTYPE_TUNNEL_GRENAT |
442			RTE_PTYPE_INNER_L2_ETHER_VLAN |
443			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
444			RTE_PTYPE_INNER_L4_SCTP,
445		[87] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
446			RTE_PTYPE_TUNNEL_GRENAT |
447			RTE_PTYPE_INNER_L2_ETHER_VLAN |
448			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
449			RTE_PTYPE_INNER_L4_ICMP,
450
451		/* Non tunneled IPv6 */
452		[88] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
453			RTE_PTYPE_L4_FRAG,
454		[89] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
455			RTE_PTYPE_L4_NONFRAG,
456		[90] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
457			RTE_PTYPE_L4_UDP,
458		/* [91] reserved */
459		[92] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
460			RTE_PTYPE_L4_TCP,
461		[93] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
462			RTE_PTYPE_L4_SCTP,
463		[94] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
464			RTE_PTYPE_L4_ICMP,
465
466		/* IPv6 --> IPv4 */
467		[95] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
468			RTE_PTYPE_TUNNEL_IP |
469			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
470			RTE_PTYPE_INNER_L4_FRAG,
471		[96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
472			RTE_PTYPE_TUNNEL_IP |
473			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
474			RTE_PTYPE_INNER_L4_NONFRAG,
475		[97] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
476			RTE_PTYPE_TUNNEL_IP |
477			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
478			RTE_PTYPE_INNER_L4_UDP,
479		/* [98] reserved */
480		[99] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
481			RTE_PTYPE_TUNNEL_IP |
482			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
483			RTE_PTYPE_INNER_L4_TCP,
484		[100] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
485			RTE_PTYPE_TUNNEL_IP |
486			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
487			RTE_PTYPE_INNER_L4_SCTP,
488		[101] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
489			RTE_PTYPE_TUNNEL_IP |
490			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
491			RTE_PTYPE_INNER_L4_ICMP,
492
493		/* IPv6 --> IPv6 */
494		[102] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
495			RTE_PTYPE_TUNNEL_IP |
496			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
497			RTE_PTYPE_INNER_L4_FRAG,
498		[103] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
499			RTE_PTYPE_TUNNEL_IP |
500			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
501			RTE_PTYPE_INNER_L4_NONFRAG,
502		[104] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
503			RTE_PTYPE_TUNNEL_IP |
504			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
505			RTE_PTYPE_INNER_L4_UDP,
506		/* [105] reserved */
507		[106] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
508			RTE_PTYPE_TUNNEL_IP |
509			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
510			RTE_PTYPE_INNER_L4_TCP,
511		[107] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
512			RTE_PTYPE_TUNNEL_IP |
513			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
514			RTE_PTYPE_INNER_L4_SCTP,
515		[108] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
516			RTE_PTYPE_TUNNEL_IP |
517			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
518			RTE_PTYPE_INNER_L4_ICMP,
519
520		/* IPv6 --> GRE/Teredo/VXLAN */
521		[109] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
522			RTE_PTYPE_TUNNEL_GRENAT,
523
524		/* IPv6 --> GRE/Teredo/VXLAN --> IPv4 */
525		[110] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
526			RTE_PTYPE_TUNNEL_GRENAT |
527			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
528			RTE_PTYPE_INNER_L4_FRAG,
529		[111] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
530			RTE_PTYPE_TUNNEL_GRENAT |
531			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
532			RTE_PTYPE_INNER_L4_NONFRAG,
533		[112] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
534			RTE_PTYPE_TUNNEL_GRENAT |
535			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
536			RTE_PTYPE_INNER_L4_UDP,
537		/* [113] reserved */
538		[114] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
539			RTE_PTYPE_TUNNEL_GRENAT |
540			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
541			RTE_PTYPE_INNER_L4_TCP,
542		[115] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
543			RTE_PTYPE_TUNNEL_GRENAT |
544			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
545			RTE_PTYPE_INNER_L4_SCTP,
546		[116] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
547			RTE_PTYPE_TUNNEL_GRENAT |
548			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
549			RTE_PTYPE_INNER_L4_ICMP,
550
551		/* IPv6 --> GRE/Teredo/VXLAN --> IPv6 */
552		[117] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
553			RTE_PTYPE_TUNNEL_GRENAT |
554			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
555			RTE_PTYPE_INNER_L4_FRAG,
556		[118] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
557			RTE_PTYPE_TUNNEL_GRENAT |
558			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
559			RTE_PTYPE_INNER_L4_NONFRAG,
560		[119] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
561			RTE_PTYPE_TUNNEL_GRENAT |
562			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
563			RTE_PTYPE_INNER_L4_UDP,
564		/* [120] reserved */
565		[121] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
566			RTE_PTYPE_TUNNEL_GRENAT |
567			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
568			RTE_PTYPE_INNER_L4_TCP,
569		[122] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
570			RTE_PTYPE_TUNNEL_GRENAT |
571			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
572			RTE_PTYPE_INNER_L4_SCTP,
573		[123] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
574			RTE_PTYPE_TUNNEL_GRENAT |
575			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
576			RTE_PTYPE_INNER_L4_ICMP,
577
578		/* IPv6 --> GRE/Teredo/VXLAN --> MAC */
579		[124] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
580			RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER,
581
582		/* IPv6 --> GRE/Teredo/VXLAN --> MAC --> IPv4 */
583		[125] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
584			RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
585			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
586			RTE_PTYPE_INNER_L4_FRAG,
587		[126] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
588			RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
589			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
590			RTE_PTYPE_INNER_L4_NONFRAG,
591		[127] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
592			RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
593			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
594			RTE_PTYPE_INNER_L4_UDP,
595		/* [128] reserved */
596		[129] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
597			RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
598			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
599			RTE_PTYPE_INNER_L4_TCP,
600		[130] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
601			RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
602			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
603			RTE_PTYPE_INNER_L4_SCTP,
604		[131] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
605			RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
606			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
607			RTE_PTYPE_INNER_L4_ICMP,
608
609		/* IPv6 --> GRE/Teredo/VXLAN --> MAC --> IPv6 */
610		[132] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
611			RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
612			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
613			RTE_PTYPE_INNER_L4_FRAG,
614		[133] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
615			RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
616			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
617			RTE_PTYPE_INNER_L4_NONFRAG,
618		[134] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
619			RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
620			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
621			RTE_PTYPE_INNER_L4_UDP,
622		/* [135] reserved */
623		[136] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
624			RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
625			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
626			RTE_PTYPE_INNER_L4_TCP,
627		[137] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
628			RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
629			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
630			RTE_PTYPE_INNER_L4_SCTP,
631		[138] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
632			RTE_PTYPE_TUNNEL_GRENAT | RTE_PTYPE_INNER_L2_ETHER |
633			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
634			RTE_PTYPE_INNER_L4_ICMP,
635
636		/* IPv6 --> GRE/Teredo/VXLAN --> MAC/VLAN */
637		[139] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
638			RTE_PTYPE_TUNNEL_GRENAT |
639			RTE_PTYPE_INNER_L2_ETHER_VLAN,
640
641		/* IPv6 --> GRE/Teredo/VXLAN --> MAC/VLAN --> IPv4 */
642		[140] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
643			RTE_PTYPE_TUNNEL_GRENAT |
644			RTE_PTYPE_INNER_L2_ETHER_VLAN |
645			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
646			RTE_PTYPE_INNER_L4_FRAG,
647		[141] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
648			RTE_PTYPE_TUNNEL_GRENAT |
649			RTE_PTYPE_INNER_L2_ETHER_VLAN |
650			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
651			RTE_PTYPE_INNER_L4_NONFRAG,
652		[142] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
653			RTE_PTYPE_TUNNEL_GRENAT |
654			RTE_PTYPE_INNER_L2_ETHER_VLAN |
655			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
656			RTE_PTYPE_INNER_L4_UDP,
657		/* [143] reserved */
658		[144] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
659			RTE_PTYPE_TUNNEL_GRENAT |
660			RTE_PTYPE_INNER_L2_ETHER_VLAN |
661			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
662			RTE_PTYPE_INNER_L4_TCP,
663		[145] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
664			RTE_PTYPE_TUNNEL_GRENAT |
665			RTE_PTYPE_INNER_L2_ETHER_VLAN |
666			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
667			RTE_PTYPE_INNER_L4_SCTP,
668		[146] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
669			RTE_PTYPE_TUNNEL_GRENAT |
670			RTE_PTYPE_INNER_L2_ETHER_VLAN |
671			RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN |
672			RTE_PTYPE_INNER_L4_ICMP,
673
674		/* IPv6 --> GRE/Teredo/VXLAN --> MAC/VLAN --> IPv6 */
675		[147] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
676			RTE_PTYPE_TUNNEL_GRENAT |
677			RTE_PTYPE_INNER_L2_ETHER_VLAN |
678			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
679			RTE_PTYPE_INNER_L4_FRAG,
680		[148] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
681			RTE_PTYPE_TUNNEL_GRENAT |
682			RTE_PTYPE_INNER_L2_ETHER_VLAN |
683			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
684			RTE_PTYPE_INNER_L4_NONFRAG,
685		[149] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
686			RTE_PTYPE_TUNNEL_GRENAT |
687			RTE_PTYPE_INNER_L2_ETHER_VLAN |
688			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
689			RTE_PTYPE_INNER_L4_UDP,
690		/* [150] reserved */
691		[151] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
692			RTE_PTYPE_TUNNEL_GRENAT |
693			RTE_PTYPE_INNER_L2_ETHER_VLAN |
694			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
695			RTE_PTYPE_INNER_L4_TCP,
696		[152] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
697			RTE_PTYPE_TUNNEL_GRENAT |
698			RTE_PTYPE_INNER_L2_ETHER_VLAN |
699			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
700			RTE_PTYPE_INNER_L4_SCTP,
701		[153] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
702			RTE_PTYPE_TUNNEL_GRENAT |
703			RTE_PTYPE_INNER_L2_ETHER_VLAN |
704			RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN |
705			RTE_PTYPE_INNER_L4_ICMP,
706
707		/* L2 NSH packet type */
708		[154] = RTE_PTYPE_L2_ETHER_NSH,
709		[155] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
710			RTE_PTYPE_L4_FRAG,
711		[156] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
712			RTE_PTYPE_L4_NONFRAG,
713		[157] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
714			RTE_PTYPE_L4_UDP,
715		[158] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
716			RTE_PTYPE_L4_TCP,
717		[159] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
718			RTE_PTYPE_L4_SCTP,
719		[160] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
720			RTE_PTYPE_L4_ICMP,
721		[161] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
722			RTE_PTYPE_L4_FRAG,
723		[162] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
724			RTE_PTYPE_L4_NONFRAG,
725		[163] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
726			RTE_PTYPE_L4_UDP,
727		[164] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
728			RTE_PTYPE_L4_TCP,
729		[165] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
730			RTE_PTYPE_L4_SCTP,
731		[166] = RTE_PTYPE_L2_ETHER_NSH | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
732			RTE_PTYPE_L4_ICMP,
733
734		/* All others reserved */
735	};
736
737	return type_table[ptype];
738}
739
740#define I40E_RX_DESC_EXT_STATUS_FLEXBH_MASK   0x03
741#define I40E_RX_DESC_EXT_STATUS_FLEXBH_FD_ID  0x01
742#define I40E_RX_DESC_EXT_STATUS_FLEXBH_FLEX   0x02
743#define I40E_RX_DESC_EXT_STATUS_FLEXBL_MASK   0x03
744#define I40E_RX_DESC_EXT_STATUS_FLEXBL_FLEX   0x01
745
746static inline uint64_t
747i40e_rxd_build_fdir(volatile union i40e_rx_desc *rxdp, struct rte_mbuf *mb)
748{
749	uint64_t flags = 0;
750#ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC
751	uint16_t flexbh, flexbl;
752
753	flexbh = (rte_le_to_cpu_32(rxdp->wb.qword2.ext_status) >>
754		I40E_RX_DESC_EXT_STATUS_FLEXBH_SHIFT) &
755		I40E_RX_DESC_EXT_STATUS_FLEXBH_MASK;
756	flexbl = (rte_le_to_cpu_32(rxdp->wb.qword2.ext_status) >>
757		I40E_RX_DESC_EXT_STATUS_FLEXBL_SHIFT) &
758		I40E_RX_DESC_EXT_STATUS_FLEXBL_MASK;
759
760
761	if (flexbh == I40E_RX_DESC_EXT_STATUS_FLEXBH_FD_ID) {
762		mb->hash.fdir.hi =
763			rte_le_to_cpu_32(rxdp->wb.qword3.hi_dword.fd_id);
764		flags |= PKT_RX_FDIR_ID;
765	} else if (flexbh == I40E_RX_DESC_EXT_STATUS_FLEXBH_FLEX) {
766		mb->hash.fdir.hi =
767			rte_le_to_cpu_32(rxdp->wb.qword3.hi_dword.flex_bytes_hi);
768		flags |= PKT_RX_FDIR_FLX;
769	}
770	if (flexbl == I40E_RX_DESC_EXT_STATUS_FLEXBL_FLEX) {
771		mb->hash.fdir.lo =
772			rte_le_to_cpu_32(rxdp->wb.qword3.lo_dword.flex_bytes_lo);
773		flags |= PKT_RX_FDIR_FLX;
774	}
775#else
776	mb->hash.fdir.hi =
777		rte_le_to_cpu_32(rxdp->wb.qword0.hi_dword.fd_id);
778	flags |= PKT_RX_FDIR_ID;
779#endif
780	return flags;
781}
782static inline void
783i40e_txd_enable_checksum(uint64_t ol_flags,
784			uint32_t *td_cmd,
785			uint32_t *td_offset,
786			union i40e_tx_offload tx_offload,
787			uint32_t *cd_tunneling)
788{
789	/* UDP tunneling packet TX checksum offload */
790	if (ol_flags & PKT_TX_OUTER_IP_CKSUM) {
791
792		*td_offset |= (tx_offload.outer_l2_len >> 1)
793				<< I40E_TX_DESC_LENGTH_MACLEN_SHIFT;
794
795		if (ol_flags & PKT_TX_OUTER_IP_CKSUM)
796			*cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV4;
797		else if (ol_flags & PKT_TX_OUTER_IPV4)
798			*cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM;
799		else if (ol_flags & PKT_TX_OUTER_IPV6)
800			*cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV6;
801
802		/* Now set the ctx descriptor fields */
803		*cd_tunneling |= (tx_offload.outer_l3_len >> 2) <<
804				I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT |
805				(tx_offload.l2_len >> 1) <<
806				I40E_TXD_CTX_QW0_NATLEN_SHIFT;
807
808	} else
809		*td_offset |= (tx_offload.l2_len >> 1)
810			<< I40E_TX_DESC_LENGTH_MACLEN_SHIFT;
811
812	/* Enable L3 checksum offloads */
813	if (ol_flags & PKT_TX_IP_CKSUM) {
814		*td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM;
815		*td_offset |= (tx_offload.l3_len >> 2)
816				<< I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
817	} else if (ol_flags & PKT_TX_IPV4) {
818		*td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4;
819		*td_offset |= (tx_offload.l3_len >> 2)
820				<< I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
821	} else if (ol_flags & PKT_TX_IPV6) {
822		*td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV6;
823		*td_offset |= (tx_offload.l3_len >> 2)
824				<< I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
825	}
826
827	if (ol_flags & PKT_TX_TCP_SEG) {
828		*td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP;
829		*td_offset |= (tx_offload.l4_len >> 2)
830			<< I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
831		return;
832	}
833
834	/* Enable L4 checksum offloads */
835	switch (ol_flags & PKT_TX_L4_MASK) {
836	case PKT_TX_TCP_CKSUM:
837		*td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP;
838		*td_offset |= (sizeof(struct tcp_hdr) >> 2) <<
839				I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
840		break;
841	case PKT_TX_SCTP_CKSUM:
842		*td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP;
843		*td_offset |= (sizeof(struct sctp_hdr) >> 2) <<
844				I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
845		break;
846	case PKT_TX_UDP_CKSUM:
847		*td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP;
848		*td_offset |= (sizeof(struct udp_hdr) >> 2) <<
849				I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
850		break;
851	default:
852		break;
853	}
854}
855
856/* Construct the tx flags */
857static inline uint64_t
858i40e_build_ctob(uint32_t td_cmd,
859		uint32_t td_offset,
860		unsigned int size,
861		uint32_t td_tag)
862{
863	return rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DATA |
864			((uint64_t)td_cmd  << I40E_TXD_QW1_CMD_SHIFT) |
865			((uint64_t)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) |
866			((uint64_t)size  << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) |
867			((uint64_t)td_tag  << I40E_TXD_QW1_L2TAG1_SHIFT));
868}
869
870static inline int
871i40e_xmit_cleanup(struct i40e_tx_queue *txq)
872{
873	struct i40e_tx_entry *sw_ring = txq->sw_ring;
874	volatile struct i40e_tx_desc *txd = txq->tx_ring;
875	uint16_t last_desc_cleaned = txq->last_desc_cleaned;
876	uint16_t nb_tx_desc = txq->nb_tx_desc;
877	uint16_t desc_to_clean_to;
878	uint16_t nb_tx_to_clean;
879
880	desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->tx_rs_thresh);
881	if (desc_to_clean_to >= nb_tx_desc)
882		desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc);
883
884	desc_to_clean_to = sw_ring[desc_to_clean_to].last_id;
885	if ((txd[desc_to_clean_to].cmd_type_offset_bsz &
886			rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) !=
887			rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE)) {
888		PMD_TX_FREE_LOG(DEBUG, "TX descriptor %4u is not done "
889			"(port=%d queue=%d)", desc_to_clean_to,
890				txq->port_id, txq->queue_id);
891		return -1;
892	}
893
894	if (last_desc_cleaned > desc_to_clean_to)
895		nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) +
896							desc_to_clean_to);
897	else
898		nb_tx_to_clean = (uint16_t)(desc_to_clean_to -
899					last_desc_cleaned);
900
901	txd[desc_to_clean_to].cmd_type_offset_bsz = 0;
902
903	txq->last_desc_cleaned = desc_to_clean_to;
904	txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean);
905
906	return 0;
907}
908
909static inline int
910#ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC
911check_rx_burst_bulk_alloc_preconditions(struct i40e_rx_queue *rxq)
912#else
913check_rx_burst_bulk_alloc_preconditions(__rte_unused struct i40e_rx_queue *rxq)
914#endif
915{
916	int ret = 0;
917
918#ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC
919	if (!(rxq->rx_free_thresh >= RTE_PMD_I40E_RX_MAX_BURST)) {
920		PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
921			     "rxq->rx_free_thresh=%d, "
922			     "RTE_PMD_I40E_RX_MAX_BURST=%d",
923			     rxq->rx_free_thresh, RTE_PMD_I40E_RX_MAX_BURST);
924		ret = -EINVAL;
925	} else if (!(rxq->rx_free_thresh < rxq->nb_rx_desc)) {
926		PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
927			     "rxq->rx_free_thresh=%d, "
928			     "rxq->nb_rx_desc=%d",
929			     rxq->rx_free_thresh, rxq->nb_rx_desc);
930		ret = -EINVAL;
931	} else if (rxq->nb_rx_desc % rxq->rx_free_thresh != 0) {
932		PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
933			     "rxq->nb_rx_desc=%d, "
934			     "rxq->rx_free_thresh=%d",
935			     rxq->nb_rx_desc, rxq->rx_free_thresh);
936		ret = -EINVAL;
937	} else if (!(rxq->nb_rx_desc < (I40E_MAX_RING_DESC -
938				RTE_PMD_I40E_RX_MAX_BURST))) {
939		PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
940			     "rxq->nb_rx_desc=%d, "
941			     "I40E_MAX_RING_DESC=%d, "
942			     "RTE_PMD_I40E_RX_MAX_BURST=%d",
943			     rxq->nb_rx_desc, I40E_MAX_RING_DESC,
944			     RTE_PMD_I40E_RX_MAX_BURST);
945		ret = -EINVAL;
946	}
947#else
948	ret = -EINVAL;
949#endif
950
951	return ret;
952}
953
954#ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC
955#define I40E_LOOK_AHEAD 8
956#if (I40E_LOOK_AHEAD != 8)
957#error "PMD I40E: I40E_LOOK_AHEAD must be 8\n"
958#endif
959static inline int
960i40e_rx_scan_hw_ring(struct i40e_rx_queue *rxq)
961{
962	volatile union i40e_rx_desc *rxdp;
963	struct i40e_rx_entry *rxep;
964	struct rte_mbuf *mb;
965	uint16_t pkt_len;
966	uint64_t qword1;
967	uint32_t rx_status;
968	int32_t s[I40E_LOOK_AHEAD], nb_dd;
969	int32_t i, j, nb_rx = 0;
970	uint64_t pkt_flags;
971
972	rxdp = &rxq->rx_ring[rxq->rx_tail];
973	rxep = &rxq->sw_ring[rxq->rx_tail];
974
975	qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len);
976	rx_status = (qword1 & I40E_RXD_QW1_STATUS_MASK) >>
977				I40E_RXD_QW1_STATUS_SHIFT;
978
979	/* Make sure there is at least 1 packet to receive */
980	if (!(rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)))
981		return 0;
982
983	/**
984	 * Scan LOOK_AHEAD descriptors at a time to determine which
985	 * descriptors reference packets that are ready to be received.
986	 */
987	for (i = 0; i < RTE_PMD_I40E_RX_MAX_BURST; i+=I40E_LOOK_AHEAD,
988			rxdp += I40E_LOOK_AHEAD, rxep += I40E_LOOK_AHEAD) {
989		/* Read desc statuses backwards to avoid race condition */
990		for (j = I40E_LOOK_AHEAD - 1; j >= 0; j--) {
991			qword1 = rte_le_to_cpu_64(\
992				rxdp[j].wb.qword1.status_error_len);
993			s[j] = (qword1 & I40E_RXD_QW1_STATUS_MASK) >>
994					I40E_RXD_QW1_STATUS_SHIFT;
995		}
996
997		/* Compute how many status bits were set */
998		for (j = 0, nb_dd = 0; j < I40E_LOOK_AHEAD; j++)
999			nb_dd += s[j] & (1 << I40E_RX_DESC_STATUS_DD_SHIFT);
1000
1001		nb_rx += nb_dd;
1002
1003		/* Translate descriptor info to mbuf parameters */
1004		for (j = 0; j < nb_dd; j++) {
1005			mb = rxep[j].mbuf;
1006			qword1 = rte_le_to_cpu_64(\
1007				rxdp[j].wb.qword1.status_error_len);
1008			pkt_len = ((qword1 & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
1009				I40E_RXD_QW1_LENGTH_PBUF_SHIFT) - rxq->crc_len;
1010			mb->data_len = pkt_len;
1011			mb->pkt_len = pkt_len;
1012			mb->ol_flags = 0;
1013			i40e_rxd_to_vlan_tci(mb, &rxdp[j]);
1014			pkt_flags = i40e_rxd_status_to_pkt_flags(qword1);
1015			pkt_flags |= i40e_rxd_error_to_pkt_flags(qword1);
1016			mb->packet_type =
1017				i40e_rxd_pkt_type_mapping((uint8_t)((qword1 &
1018						I40E_RXD_QW1_PTYPE_MASK) >>
1019						I40E_RXD_QW1_PTYPE_SHIFT));
1020			if (pkt_flags & PKT_RX_RSS_HASH)
1021				mb->hash.rss = rte_le_to_cpu_32(\
1022					rxdp[j].wb.qword0.hi_dword.rss);
1023			if (pkt_flags & PKT_RX_FDIR)
1024				pkt_flags |= i40e_rxd_build_fdir(&rxdp[j], mb);
1025
1026#ifdef RTE_LIBRTE_IEEE1588
1027			pkt_flags |= i40e_get_iee15888_flags(mb, qword1);
1028#endif
1029			mb->ol_flags |= pkt_flags;
1030
1031		}
1032
1033		for (j = 0; j < I40E_LOOK_AHEAD; j++)
1034			rxq->rx_stage[i + j] = rxep[j].mbuf;
1035
1036		if (nb_dd != I40E_LOOK_AHEAD)
1037			break;
1038	}
1039
1040	/* Clear software ring entries */
1041	for (i = 0; i < nb_rx; i++)
1042		rxq->sw_ring[rxq->rx_tail + i].mbuf = NULL;
1043
1044	return nb_rx;
1045}
1046
1047static inline uint16_t
1048i40e_rx_fill_from_stage(struct i40e_rx_queue *rxq,
1049			struct rte_mbuf **rx_pkts,
1050			uint16_t nb_pkts)
1051{
1052	uint16_t i;
1053	struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail];
1054
1055	nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail);
1056
1057	for (i = 0; i < nb_pkts; i++)
1058		rx_pkts[i] = stage[i];
1059
1060	rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts);
1061	rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts);
1062
1063	return nb_pkts;
1064}
1065
1066static inline int
1067i40e_rx_alloc_bufs(struct i40e_rx_queue *rxq)
1068{
1069	volatile union i40e_rx_desc *rxdp;
1070	struct i40e_rx_entry *rxep;
1071	struct rte_mbuf *mb;
1072	uint16_t alloc_idx, i;
1073	uint64_t dma_addr;
1074	int diag;
1075
1076	/* Allocate buffers in bulk */
1077	alloc_idx = (uint16_t)(rxq->rx_free_trigger -
1078				(rxq->rx_free_thresh - 1));
1079	rxep = &(rxq->sw_ring[alloc_idx]);
1080	diag = rte_mempool_get_bulk(rxq->mp, (void *)rxep,
1081					rxq->rx_free_thresh);
1082	if (unlikely(diag != 0)) {
1083		PMD_DRV_LOG(ERR, "Failed to get mbufs in bulk");
1084		return -ENOMEM;
1085	}
1086
1087	rxdp = &rxq->rx_ring[alloc_idx];
1088	for (i = 0; i < rxq->rx_free_thresh; i++) {
1089		if (likely(i < (rxq->rx_free_thresh - 1)))
1090			/* Prefetch next mbuf */
1091			rte_prefetch0(rxep[i + 1].mbuf);
1092
1093		mb = rxep[i].mbuf;
1094		rte_mbuf_refcnt_set(mb, 1);
1095		mb->next = NULL;
1096		mb->data_off = RTE_PKTMBUF_HEADROOM;
1097		mb->nb_segs = 1;
1098		mb->port = rxq->port_id;
1099		dma_addr = rte_cpu_to_le_64(\
1100			rte_mbuf_data_dma_addr_default(mb));
1101		rxdp[i].read.hdr_addr = 0;
1102		rxdp[i].read.pkt_addr = dma_addr;
1103	}
1104
1105	/* Update rx tail regsiter */
1106	rte_wmb();
1107	I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->rx_free_trigger);
1108
1109	rxq->rx_free_trigger =
1110		(uint16_t)(rxq->rx_free_trigger + rxq->rx_free_thresh);
1111	if (rxq->rx_free_trigger >= rxq->nb_rx_desc)
1112		rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1);
1113
1114	return 0;
1115}
1116
1117static inline uint16_t
1118rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
1119{
1120	struct i40e_rx_queue *rxq = (struct i40e_rx_queue *)rx_queue;
1121	uint16_t nb_rx = 0;
1122
1123	if (!nb_pkts)
1124		return 0;
1125
1126	if (rxq->rx_nb_avail)
1127		return i40e_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1128
1129	nb_rx = (uint16_t)i40e_rx_scan_hw_ring(rxq);
1130	rxq->rx_next_avail = 0;
1131	rxq->rx_nb_avail = nb_rx;
1132	rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx);
1133
1134	if (rxq->rx_tail > rxq->rx_free_trigger) {
1135		if (i40e_rx_alloc_bufs(rxq) != 0) {
1136			uint16_t i, j;
1137
1138			PMD_RX_LOG(DEBUG, "Rx mbuf alloc failed for "
1139				   "port_id=%u, queue_id=%u",
1140				   rxq->port_id, rxq->queue_id);
1141			rxq->rx_nb_avail = 0;
1142			rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx);
1143			for (i = 0, j = rxq->rx_tail; i < nb_rx; i++, j++)
1144				rxq->sw_ring[j].mbuf = rxq->rx_stage[i];
1145
1146			return 0;
1147		}
1148	}
1149
1150	if (rxq->rx_tail >= rxq->nb_rx_desc)
1151		rxq->rx_tail = 0;
1152
1153	if (rxq->rx_nb_avail)
1154		return i40e_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
1155
1156	return 0;
1157}
1158
1159static uint16_t
1160i40e_recv_pkts_bulk_alloc(void *rx_queue,
1161			  struct rte_mbuf **rx_pkts,
1162			  uint16_t nb_pkts)
1163{
1164	uint16_t nb_rx = 0, n, count;
1165
1166	if (unlikely(nb_pkts == 0))
1167		return 0;
1168
1169	if (likely(nb_pkts <= RTE_PMD_I40E_RX_MAX_BURST))
1170		return rx_recv_pkts(rx_queue, rx_pkts, nb_pkts);
1171
1172	while (nb_pkts) {
1173		n = RTE_MIN(nb_pkts, RTE_PMD_I40E_RX_MAX_BURST);
1174		count = rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n);
1175		nb_rx = (uint16_t)(nb_rx + count);
1176		nb_pkts = (uint16_t)(nb_pkts - count);
1177		if (count < n)
1178			break;
1179	}
1180
1181	return nb_rx;
1182}
1183#else
1184static uint16_t
1185i40e_recv_pkts_bulk_alloc(void __rte_unused *rx_queue,
1186			  struct rte_mbuf __rte_unused **rx_pkts,
1187			  uint16_t __rte_unused nb_pkts)
1188{
1189	return 0;
1190}
1191#endif /* RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC */
1192
1193uint16_t
1194i40e_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
1195{
1196	struct i40e_rx_queue *rxq;
1197	volatile union i40e_rx_desc *rx_ring;
1198	volatile union i40e_rx_desc *rxdp;
1199	union i40e_rx_desc rxd;
1200	struct i40e_rx_entry *sw_ring;
1201	struct i40e_rx_entry *rxe;
1202	struct rte_mbuf *rxm;
1203	struct rte_mbuf *nmb;
1204	uint16_t nb_rx;
1205	uint32_t rx_status;
1206	uint64_t qword1;
1207	uint16_t rx_packet_len;
1208	uint16_t rx_id, nb_hold;
1209	uint64_t dma_addr;
1210	uint64_t pkt_flags;
1211
1212	nb_rx = 0;
1213	nb_hold = 0;
1214	rxq = rx_queue;
1215	rx_id = rxq->rx_tail;
1216	rx_ring = rxq->rx_ring;
1217	sw_ring = rxq->sw_ring;
1218
1219	while (nb_rx < nb_pkts) {
1220		rxdp = &rx_ring[rx_id];
1221		qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len);
1222		rx_status = (qword1 & I40E_RXD_QW1_STATUS_MASK)
1223				>> I40E_RXD_QW1_STATUS_SHIFT;
1224
1225		/* Check the DD bit first */
1226		if (!(rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)))
1227			break;
1228
1229		nmb = rte_mbuf_raw_alloc(rxq->mp);
1230		if (unlikely(!nmb))
1231			break;
1232		rxd = *rxdp;
1233
1234		nb_hold++;
1235		rxe = &sw_ring[rx_id];
1236		rx_id++;
1237		if (unlikely(rx_id == rxq->nb_rx_desc))
1238			rx_id = 0;
1239
1240		/* Prefetch next mbuf */
1241		rte_prefetch0(sw_ring[rx_id].mbuf);
1242
1243		/**
1244		 * When next RX descriptor is on a cache line boundary,
1245		 * prefetch the next 4 RX descriptors and next 8 pointers
1246		 * to mbufs.
1247		 */
1248		if ((rx_id & 0x3) == 0) {
1249			rte_prefetch0(&rx_ring[rx_id]);
1250			rte_prefetch0(&sw_ring[rx_id]);
1251		}
1252		rxm = rxe->mbuf;
1253		rxe->mbuf = nmb;
1254		dma_addr =
1255			rte_cpu_to_le_64(rte_mbuf_data_dma_addr_default(nmb));
1256		rxdp->read.hdr_addr = 0;
1257		rxdp->read.pkt_addr = dma_addr;
1258
1259		rx_packet_len = ((qword1 & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
1260				I40E_RXD_QW1_LENGTH_PBUF_SHIFT) - rxq->crc_len;
1261
1262		rxm->data_off = RTE_PKTMBUF_HEADROOM;
1263		rte_prefetch0(RTE_PTR_ADD(rxm->buf_addr, RTE_PKTMBUF_HEADROOM));
1264		rxm->nb_segs = 1;
1265		rxm->next = NULL;
1266		rxm->pkt_len = rx_packet_len;
1267		rxm->data_len = rx_packet_len;
1268		rxm->port = rxq->port_id;
1269		rxm->ol_flags = 0;
1270		i40e_rxd_to_vlan_tci(rxm, &rxd);
1271		pkt_flags = i40e_rxd_status_to_pkt_flags(qword1);
1272		pkt_flags |= i40e_rxd_error_to_pkt_flags(qword1);
1273		rxm->packet_type =
1274			i40e_rxd_pkt_type_mapping((uint8_t)((qword1 &
1275			I40E_RXD_QW1_PTYPE_MASK) >> I40E_RXD_QW1_PTYPE_SHIFT));
1276		if (pkt_flags & PKT_RX_RSS_HASH)
1277			rxm->hash.rss =
1278				rte_le_to_cpu_32(rxd.wb.qword0.hi_dword.rss);
1279		if (pkt_flags & PKT_RX_FDIR)
1280			pkt_flags |= i40e_rxd_build_fdir(&rxd, rxm);
1281
1282#ifdef RTE_LIBRTE_IEEE1588
1283		pkt_flags |= i40e_get_iee15888_flags(rxm, qword1);
1284#endif
1285		rxm->ol_flags |= pkt_flags;
1286
1287		rx_pkts[nb_rx++] = rxm;
1288	}
1289	rxq->rx_tail = rx_id;
1290
1291	/**
1292	 * If the number of free RX descriptors is greater than the RX free
1293	 * threshold of the queue, advance the receive tail register of queue.
1294	 * Update that register with the value of the last processed RX
1295	 * descriptor minus 1.
1296	 */
1297	nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold);
1298	if (nb_hold > rxq->rx_free_thresh) {
1299		rx_id = (uint16_t) ((rx_id == 0) ?
1300			(rxq->nb_rx_desc - 1) : (rx_id - 1));
1301		I40E_PCI_REG_WRITE(rxq->qrx_tail, rx_id);
1302		nb_hold = 0;
1303	}
1304	rxq->nb_rx_hold = nb_hold;
1305
1306	return nb_rx;
1307}
1308
1309uint16_t
1310i40e_recv_scattered_pkts(void *rx_queue,
1311			 struct rte_mbuf **rx_pkts,
1312			 uint16_t nb_pkts)
1313{
1314	struct i40e_rx_queue *rxq = rx_queue;
1315	volatile union i40e_rx_desc *rx_ring = rxq->rx_ring;
1316	volatile union i40e_rx_desc *rxdp;
1317	union i40e_rx_desc rxd;
1318	struct i40e_rx_entry *sw_ring = rxq->sw_ring;
1319	struct i40e_rx_entry *rxe;
1320	struct rte_mbuf *first_seg = rxq->pkt_first_seg;
1321	struct rte_mbuf *last_seg = rxq->pkt_last_seg;
1322	struct rte_mbuf *nmb, *rxm;
1323	uint16_t rx_id = rxq->rx_tail;
1324	uint16_t nb_rx = 0, nb_hold = 0, rx_packet_len;
1325	uint32_t rx_status;
1326	uint64_t qword1;
1327	uint64_t dma_addr;
1328	uint64_t pkt_flags;
1329
1330	while (nb_rx < nb_pkts) {
1331		rxdp = &rx_ring[rx_id];
1332		qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len);
1333		rx_status = (qword1 & I40E_RXD_QW1_STATUS_MASK) >>
1334					I40E_RXD_QW1_STATUS_SHIFT;
1335
1336		/* Check the DD bit */
1337		if (!(rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)))
1338			break;
1339
1340		nmb = rte_mbuf_raw_alloc(rxq->mp);
1341		if (unlikely(!nmb))
1342			break;
1343		rxd = *rxdp;
1344		nb_hold++;
1345		rxe = &sw_ring[rx_id];
1346		rx_id++;
1347		if (rx_id == rxq->nb_rx_desc)
1348			rx_id = 0;
1349
1350		/* Prefetch next mbuf */
1351		rte_prefetch0(sw_ring[rx_id].mbuf);
1352
1353		/**
1354		 * When next RX descriptor is on a cache line boundary,
1355		 * prefetch the next 4 RX descriptors and next 8 pointers
1356		 * to mbufs.
1357		 */
1358		if ((rx_id & 0x3) == 0) {
1359			rte_prefetch0(&rx_ring[rx_id]);
1360			rte_prefetch0(&sw_ring[rx_id]);
1361		}
1362
1363		rxm = rxe->mbuf;
1364		rxe->mbuf = nmb;
1365		dma_addr =
1366			rte_cpu_to_le_64(rte_mbuf_data_dma_addr_default(nmb));
1367
1368		/* Set data buffer address and data length of the mbuf */
1369		rxdp->read.hdr_addr = 0;
1370		rxdp->read.pkt_addr = dma_addr;
1371		rx_packet_len = (qword1 & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
1372					I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
1373		rxm->data_len = rx_packet_len;
1374		rxm->data_off = RTE_PKTMBUF_HEADROOM;
1375
1376		/**
1377		 * If this is the first buffer of the received packet, set the
1378		 * pointer to the first mbuf of the packet and initialize its
1379		 * context. Otherwise, update the total length and the number
1380		 * of segments of the current scattered packet, and update the
1381		 * pointer to the last mbuf of the current packet.
1382		 */
1383		if (!first_seg) {
1384			first_seg = rxm;
1385			first_seg->nb_segs = 1;
1386			first_seg->pkt_len = rx_packet_len;
1387		} else {
1388			first_seg->pkt_len =
1389				(uint16_t)(first_seg->pkt_len +
1390						rx_packet_len);
1391			first_seg->nb_segs++;
1392			last_seg->next = rxm;
1393		}
1394
1395		/**
1396		 * If this is not the last buffer of the received packet,
1397		 * update the pointer to the last mbuf of the current scattered
1398		 * packet and continue to parse the RX ring.
1399		 */
1400		if (!(rx_status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT))) {
1401			last_seg = rxm;
1402			continue;
1403		}
1404
1405		/**
1406		 * This is the last buffer of the received packet. If the CRC
1407		 * is not stripped by the hardware:
1408		 *  - Subtract the CRC length from the total packet length.
1409		 *  - If the last buffer only contains the whole CRC or a part
1410		 *  of it, free the mbuf associated to the last buffer. If part
1411		 *  of the CRC is also contained in the previous mbuf, subtract
1412		 *  the length of that CRC part from the data length of the
1413		 *  previous mbuf.
1414		 */
1415		rxm->next = NULL;
1416		if (unlikely(rxq->crc_len > 0)) {
1417			first_seg->pkt_len -= ETHER_CRC_LEN;
1418			if (rx_packet_len <= ETHER_CRC_LEN) {
1419				rte_pktmbuf_free_seg(rxm);
1420				first_seg->nb_segs--;
1421				last_seg->data_len =
1422					(uint16_t)(last_seg->data_len -
1423					(ETHER_CRC_LEN - rx_packet_len));
1424				last_seg->next = NULL;
1425			} else
1426				rxm->data_len = (uint16_t)(rx_packet_len -
1427								ETHER_CRC_LEN);
1428		}
1429
1430		first_seg->port = rxq->port_id;
1431		first_seg->ol_flags = 0;
1432		i40e_rxd_to_vlan_tci(first_seg, &rxd);
1433		pkt_flags = i40e_rxd_status_to_pkt_flags(qword1);
1434		pkt_flags |= i40e_rxd_error_to_pkt_flags(qword1);
1435		first_seg->packet_type =
1436			i40e_rxd_pkt_type_mapping((uint8_t)((qword1 &
1437			I40E_RXD_QW1_PTYPE_MASK) >> I40E_RXD_QW1_PTYPE_SHIFT));
1438		if (pkt_flags & PKT_RX_RSS_HASH)
1439			first_seg->hash.rss =
1440				rte_le_to_cpu_32(rxd.wb.qword0.hi_dword.rss);
1441		if (pkt_flags & PKT_RX_FDIR)
1442			pkt_flags |= i40e_rxd_build_fdir(&rxd, first_seg);
1443
1444#ifdef RTE_LIBRTE_IEEE1588
1445		pkt_flags |= i40e_get_iee15888_flags(first_seg, qword1);
1446#endif
1447		first_seg->ol_flags |= pkt_flags;
1448
1449		/* Prefetch data of first segment, if configured to do so. */
1450		rte_prefetch0(RTE_PTR_ADD(first_seg->buf_addr,
1451			first_seg->data_off));
1452		rx_pkts[nb_rx++] = first_seg;
1453		first_seg = NULL;
1454	}
1455
1456	/* Record index of the next RX descriptor to probe. */
1457	rxq->rx_tail = rx_id;
1458	rxq->pkt_first_seg = first_seg;
1459	rxq->pkt_last_seg = last_seg;
1460
1461	/**
1462	 * If the number of free RX descriptors is greater than the RX free
1463	 * threshold of the queue, advance the Receive Descriptor Tail (RDT)
1464	 * register. Update the RDT with the value of the last processed RX
1465	 * descriptor minus 1, to guarantee that the RDT register is never
1466	 * equal to the RDH register, which creates a "full" ring situtation
1467	 * from the hardware point of view.
1468	 */
1469	nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold);
1470	if (nb_hold > rxq->rx_free_thresh) {
1471		rx_id = (uint16_t)(rx_id == 0 ?
1472			(rxq->nb_rx_desc - 1) : (rx_id - 1));
1473		I40E_PCI_REG_WRITE(rxq->qrx_tail, rx_id);
1474		nb_hold = 0;
1475	}
1476	rxq->nb_rx_hold = nb_hold;
1477
1478	return nb_rx;
1479}
1480
1481/* Check if the context descriptor is needed for TX offloading */
1482static inline uint16_t
1483i40e_calc_context_desc(uint64_t flags)
1484{
1485	static uint64_t mask = PKT_TX_OUTER_IP_CKSUM |
1486		PKT_TX_TCP_SEG |
1487		PKT_TX_QINQ_PKT;
1488
1489#ifdef RTE_LIBRTE_IEEE1588
1490	mask |= PKT_TX_IEEE1588_TMST;
1491#endif
1492
1493	return (flags & mask) ? 1 : 0;
1494}
1495
1496/* set i40e TSO context descriptor */
1497static inline uint64_t
1498i40e_set_tso_ctx(struct rte_mbuf *mbuf, union i40e_tx_offload tx_offload)
1499{
1500	uint64_t ctx_desc = 0;
1501	uint32_t cd_cmd, hdr_len, cd_tso_len;
1502
1503	if (!tx_offload.l4_len) {
1504		PMD_DRV_LOG(DEBUG, "L4 length set to 0");
1505		return ctx_desc;
1506	}
1507
1508	/**
1509	 * in case of tunneling packet, the outer_l2_len and
1510	 * outer_l3_len must be 0.
1511	 */
1512	hdr_len = tx_offload.outer_l2_len +
1513		tx_offload.outer_l3_len +
1514		tx_offload.l2_len +
1515		tx_offload.l3_len +
1516		tx_offload.l4_len;
1517
1518	cd_cmd = I40E_TX_CTX_DESC_TSO;
1519	cd_tso_len = mbuf->pkt_len - hdr_len;
1520	ctx_desc |= ((uint64_t)cd_cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) |
1521		((uint64_t)cd_tso_len <<
1522		 I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) |
1523		((uint64_t)mbuf->tso_segsz <<
1524		 I40E_TXD_CTX_QW1_MSS_SHIFT);
1525
1526	return ctx_desc;
1527}
1528
1529uint16_t
1530i40e_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
1531{
1532	struct i40e_tx_queue *txq;
1533	struct i40e_tx_entry *sw_ring;
1534	struct i40e_tx_entry *txe, *txn;
1535	volatile struct i40e_tx_desc *txd;
1536	volatile struct i40e_tx_desc *txr;
1537	struct rte_mbuf *tx_pkt;
1538	struct rte_mbuf *m_seg;
1539	uint32_t cd_tunneling_params;
1540	uint16_t tx_id;
1541	uint16_t nb_tx;
1542	uint32_t td_cmd;
1543	uint32_t td_offset;
1544	uint32_t tx_flags;
1545	uint32_t td_tag;
1546	uint64_t ol_flags;
1547	uint16_t nb_used;
1548	uint16_t nb_ctx;
1549	uint16_t tx_last;
1550	uint16_t slen;
1551	uint64_t buf_dma_addr;
1552	union i40e_tx_offload tx_offload = {0};
1553
1554	txq = tx_queue;
1555	sw_ring = txq->sw_ring;
1556	txr = txq->tx_ring;
1557	tx_id = txq->tx_tail;
1558	txe = &sw_ring[tx_id];
1559
1560	/* Check if the descriptor ring needs to be cleaned. */
1561	if (txq->nb_tx_free < txq->tx_free_thresh)
1562		i40e_xmit_cleanup(txq);
1563
1564	for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
1565		td_cmd = 0;
1566		td_tag = 0;
1567		td_offset = 0;
1568		tx_flags = 0;
1569
1570		tx_pkt = *tx_pkts++;
1571		RTE_MBUF_PREFETCH_TO_FREE(txe->mbuf);
1572
1573		ol_flags = tx_pkt->ol_flags;
1574		tx_offload.l2_len = tx_pkt->l2_len;
1575		tx_offload.l3_len = tx_pkt->l3_len;
1576		tx_offload.outer_l2_len = tx_pkt->outer_l2_len;
1577		tx_offload.outer_l3_len = tx_pkt->outer_l3_len;
1578		tx_offload.l4_len = tx_pkt->l4_len;
1579		tx_offload.tso_segsz = tx_pkt->tso_segsz;
1580
1581		/* Calculate the number of context descriptors needed. */
1582		nb_ctx = i40e_calc_context_desc(ol_flags);
1583
1584		/**
1585		 * The number of descriptors that must be allocated for
1586		 * a packet equals to the number of the segments of that
1587		 * packet plus 1 context descriptor if needed.
1588		 */
1589		nb_used = (uint16_t)(tx_pkt->nb_segs + nb_ctx);
1590		tx_last = (uint16_t)(tx_id + nb_used - 1);
1591
1592		/* Circular ring */
1593		if (tx_last >= txq->nb_tx_desc)
1594			tx_last = (uint16_t)(tx_last - txq->nb_tx_desc);
1595
1596		if (nb_used > txq->nb_tx_free) {
1597			if (i40e_xmit_cleanup(txq) != 0) {
1598				if (nb_tx == 0)
1599					return 0;
1600				goto end_of_tx;
1601			}
1602			if (unlikely(nb_used > txq->tx_rs_thresh)) {
1603				while (nb_used > txq->nb_tx_free) {
1604					if (i40e_xmit_cleanup(txq) != 0) {
1605						if (nb_tx == 0)
1606							return 0;
1607						goto end_of_tx;
1608					}
1609				}
1610			}
1611		}
1612
1613		/* Descriptor based VLAN insertion */
1614		if (ol_flags & (PKT_TX_VLAN_PKT | PKT_TX_QINQ_PKT)) {
1615			tx_flags |= tx_pkt->vlan_tci <<
1616				I40E_TX_FLAG_L2TAG1_SHIFT;
1617			tx_flags |= I40E_TX_FLAG_INSERT_VLAN;
1618			td_cmd |= I40E_TX_DESC_CMD_IL2TAG1;
1619			td_tag = (tx_flags & I40E_TX_FLAG_L2TAG1_MASK) >>
1620						I40E_TX_FLAG_L2TAG1_SHIFT;
1621		}
1622
1623		/* Always enable CRC offload insertion */
1624		td_cmd |= I40E_TX_DESC_CMD_ICRC;
1625
1626		/* Enable checksum offloading */
1627		cd_tunneling_params = 0;
1628		if (ol_flags & I40E_TX_CKSUM_OFFLOAD_MASK) {
1629			i40e_txd_enable_checksum(ol_flags, &td_cmd, &td_offset,
1630				tx_offload, &cd_tunneling_params);
1631		}
1632
1633		if (nb_ctx) {
1634			/* Setup TX context descriptor if required */
1635			volatile struct i40e_tx_context_desc *ctx_txd =
1636				(volatile struct i40e_tx_context_desc *)\
1637							&txr[tx_id];
1638			uint16_t cd_l2tag2 = 0;
1639			uint64_t cd_type_cmd_tso_mss =
1640				I40E_TX_DESC_DTYPE_CONTEXT;
1641
1642			txn = &sw_ring[txe->next_id];
1643			RTE_MBUF_PREFETCH_TO_FREE(txn->mbuf);
1644			if (txe->mbuf != NULL) {
1645				rte_pktmbuf_free_seg(txe->mbuf);
1646				txe->mbuf = NULL;
1647			}
1648
1649			/* TSO enabled means no timestamp */
1650			if (ol_flags & PKT_TX_TCP_SEG)
1651				cd_type_cmd_tso_mss |=
1652					i40e_set_tso_ctx(tx_pkt, tx_offload);
1653			else {
1654#ifdef RTE_LIBRTE_IEEE1588
1655				if (ol_flags & PKT_TX_IEEE1588_TMST)
1656					cd_type_cmd_tso_mss |=
1657						((uint64_t)I40E_TX_CTX_DESC_TSYN <<
1658						 I40E_TXD_CTX_QW1_CMD_SHIFT);
1659#endif
1660			}
1661
1662			ctx_txd->tunneling_params =
1663				rte_cpu_to_le_32(cd_tunneling_params);
1664			if (ol_flags & PKT_TX_QINQ_PKT) {
1665				cd_l2tag2 = tx_pkt->vlan_tci_outer;
1666				cd_type_cmd_tso_mss |=
1667					((uint64_t)I40E_TX_CTX_DESC_IL2TAG2 <<
1668						I40E_TXD_CTX_QW1_CMD_SHIFT);
1669			}
1670			ctx_txd->l2tag2 = rte_cpu_to_le_16(cd_l2tag2);
1671			ctx_txd->type_cmd_tso_mss =
1672				rte_cpu_to_le_64(cd_type_cmd_tso_mss);
1673
1674			PMD_TX_LOG(DEBUG, "mbuf: %p, TCD[%u]:\n"
1675				"tunneling_params: %#x;\n"
1676				"l2tag2: %#hx;\n"
1677				"rsvd: %#hx;\n"
1678				"type_cmd_tso_mss: %#"PRIx64";\n",
1679				tx_pkt, tx_id,
1680				ctx_txd->tunneling_params,
1681				ctx_txd->l2tag2,
1682				ctx_txd->rsvd,
1683				ctx_txd->type_cmd_tso_mss);
1684
1685			txe->last_id = tx_last;
1686			tx_id = txe->next_id;
1687			txe = txn;
1688		}
1689
1690		m_seg = tx_pkt;
1691		do {
1692			txd = &txr[tx_id];
1693			txn = &sw_ring[txe->next_id];
1694
1695			if (txe->mbuf)
1696				rte_pktmbuf_free_seg(txe->mbuf);
1697			txe->mbuf = m_seg;
1698
1699			/* Setup TX Descriptor */
1700			slen = m_seg->data_len;
1701			buf_dma_addr = rte_mbuf_data_dma_addr(m_seg);
1702
1703			PMD_TX_LOG(DEBUG, "mbuf: %p, TDD[%u]:\n"
1704				"buf_dma_addr: %#"PRIx64";\n"
1705				"td_cmd: %#x;\n"
1706				"td_offset: %#x;\n"
1707				"td_len: %u;\n"
1708				"td_tag: %#x;\n",
1709				tx_pkt, tx_id, buf_dma_addr,
1710				td_cmd, td_offset, slen, td_tag);
1711
1712			txd->buffer_addr = rte_cpu_to_le_64(buf_dma_addr);
1713			txd->cmd_type_offset_bsz = i40e_build_ctob(td_cmd,
1714						td_offset, slen, td_tag);
1715			txe->last_id = tx_last;
1716			tx_id = txe->next_id;
1717			txe = txn;
1718			m_seg = m_seg->next;
1719		} while (m_seg != NULL);
1720
1721		/* The last packet data descriptor needs End Of Packet (EOP) */
1722		td_cmd |= I40E_TX_DESC_CMD_EOP;
1723		txq->nb_tx_used = (uint16_t)(txq->nb_tx_used + nb_used);
1724		txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_used);
1725
1726		if (txq->nb_tx_used >= txq->tx_rs_thresh) {
1727			PMD_TX_FREE_LOG(DEBUG,
1728					"Setting RS bit on TXD id="
1729					"%4u (port=%d queue=%d)",
1730					tx_last, txq->port_id, txq->queue_id);
1731
1732			td_cmd |= I40E_TX_DESC_CMD_RS;
1733
1734			/* Update txq RS bit counters */
1735			txq->nb_tx_used = 0;
1736		}
1737
1738		txd->cmd_type_offset_bsz |=
1739			rte_cpu_to_le_64(((uint64_t)td_cmd) <<
1740					I40E_TXD_QW1_CMD_SHIFT);
1741	}
1742
1743end_of_tx:
1744	rte_wmb();
1745
1746	PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u",
1747		   (unsigned) txq->port_id, (unsigned) txq->queue_id,
1748		   (unsigned) tx_id, (unsigned) nb_tx);
1749
1750	I40E_PCI_REG_WRITE(txq->qtx_tail, tx_id);
1751	txq->tx_tail = tx_id;
1752
1753	return nb_tx;
1754}
1755
1756static inline int __attribute__((always_inline))
1757i40e_tx_free_bufs(struct i40e_tx_queue *txq)
1758{
1759	struct i40e_tx_entry *txep;
1760	uint16_t i;
1761
1762	if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &
1763			rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) !=
1764			rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE))
1765		return 0;
1766
1767	txep = &(txq->sw_ring[txq->tx_next_dd - (txq->tx_rs_thresh - 1)]);
1768
1769	for (i = 0; i < txq->tx_rs_thresh; i++)
1770		rte_prefetch0((txep + i)->mbuf);
1771
1772	if (txq->txq_flags & (uint32_t)ETH_TXQ_FLAGS_NOREFCOUNT) {
1773		for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) {
1774			rte_mempool_put(txep->mbuf->pool, txep->mbuf);
1775			txep->mbuf = NULL;
1776		}
1777	} else {
1778		for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) {
1779			rte_pktmbuf_free_seg(txep->mbuf);
1780			txep->mbuf = NULL;
1781		}
1782	}
1783
1784	txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh);
1785	txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh);
1786	if (txq->tx_next_dd >= txq->nb_tx_desc)
1787		txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
1788
1789	return txq->tx_rs_thresh;
1790}
1791
1792/* Populate 4 descriptors with data from 4 mbufs */
1793static inline void
1794tx4(volatile struct i40e_tx_desc *txdp, struct rte_mbuf **pkts)
1795{
1796	uint64_t dma_addr;
1797	uint32_t i;
1798
1799	for (i = 0; i < 4; i++, txdp++, pkts++) {
1800		dma_addr = rte_mbuf_data_dma_addr(*pkts);
1801		txdp->buffer_addr = rte_cpu_to_le_64(dma_addr);
1802		txdp->cmd_type_offset_bsz =
1803			i40e_build_ctob((uint32_t)I40E_TD_CMD, 0,
1804					(*pkts)->data_len, 0);
1805	}
1806}
1807
1808/* Populate 1 descriptor with data from 1 mbuf */
1809static inline void
1810tx1(volatile struct i40e_tx_desc *txdp, struct rte_mbuf **pkts)
1811{
1812	uint64_t dma_addr;
1813
1814	dma_addr = rte_mbuf_data_dma_addr(*pkts);
1815	txdp->buffer_addr = rte_cpu_to_le_64(dma_addr);
1816	txdp->cmd_type_offset_bsz =
1817		i40e_build_ctob((uint32_t)I40E_TD_CMD, 0,
1818				(*pkts)->data_len, 0);
1819}
1820
1821/* Fill hardware descriptor ring with mbuf data */
1822static inline void
1823i40e_tx_fill_hw_ring(struct i40e_tx_queue *txq,
1824		     struct rte_mbuf **pkts,
1825		     uint16_t nb_pkts)
1826{
1827	volatile struct i40e_tx_desc *txdp = &(txq->tx_ring[txq->tx_tail]);
1828	struct i40e_tx_entry *txep = &(txq->sw_ring[txq->tx_tail]);
1829	const int N_PER_LOOP = 4;
1830	const int N_PER_LOOP_MASK = N_PER_LOOP - 1;
1831	int mainpart, leftover;
1832	int i, j;
1833
1834	mainpart = (nb_pkts & ((uint32_t) ~N_PER_LOOP_MASK));
1835	leftover = (nb_pkts & ((uint32_t)  N_PER_LOOP_MASK));
1836	for (i = 0; i < mainpart; i += N_PER_LOOP) {
1837		for (j = 0; j < N_PER_LOOP; ++j) {
1838			(txep + i + j)->mbuf = *(pkts + i + j);
1839		}
1840		tx4(txdp + i, pkts + i);
1841	}
1842	if (unlikely(leftover > 0)) {
1843		for (i = 0; i < leftover; ++i) {
1844			(txep + mainpart + i)->mbuf = *(pkts + mainpart + i);
1845			tx1(txdp + mainpart + i, pkts + mainpart + i);
1846		}
1847	}
1848}
1849
1850static inline uint16_t
1851tx_xmit_pkts(struct i40e_tx_queue *txq,
1852	     struct rte_mbuf **tx_pkts,
1853	     uint16_t nb_pkts)
1854{
1855	volatile struct i40e_tx_desc *txr = txq->tx_ring;
1856	uint16_t n = 0;
1857
1858	/**
1859	 * Begin scanning the H/W ring for done descriptors when the number
1860	 * of available descriptors drops below tx_free_thresh. For each done
1861	 * descriptor, free the associated buffer.
1862	 */
1863	if (txq->nb_tx_free < txq->tx_free_thresh)
1864		i40e_tx_free_bufs(txq);
1865
1866	/* Use available descriptor only */
1867	nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts);
1868	if (unlikely(!nb_pkts))
1869		return 0;
1870
1871	txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts);
1872	if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) {
1873		n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail);
1874		i40e_tx_fill_hw_ring(txq, tx_pkts, n);
1875		txr[txq->tx_next_rs].cmd_type_offset_bsz |=
1876			rte_cpu_to_le_64(((uint64_t)I40E_TX_DESC_CMD_RS) <<
1877						I40E_TXD_QW1_CMD_SHIFT);
1878		txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
1879		txq->tx_tail = 0;
1880	}
1881
1882	/* Fill hardware descriptor ring with mbuf data */
1883	i40e_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n));
1884	txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n));
1885
1886	/* Determin if RS bit needs to be set */
1887	if (txq->tx_tail > txq->tx_next_rs) {
1888		txr[txq->tx_next_rs].cmd_type_offset_bsz |=
1889			rte_cpu_to_le_64(((uint64_t)I40E_TX_DESC_CMD_RS) <<
1890						I40E_TXD_QW1_CMD_SHIFT);
1891		txq->tx_next_rs =
1892			(uint16_t)(txq->tx_next_rs + txq->tx_rs_thresh);
1893		if (txq->tx_next_rs >= txq->nb_tx_desc)
1894			txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
1895	}
1896
1897	if (txq->tx_tail >= txq->nb_tx_desc)
1898		txq->tx_tail = 0;
1899
1900	/* Update the tx tail register */
1901	rte_wmb();
1902	I40E_PCI_REG_WRITE(txq->qtx_tail, txq->tx_tail);
1903
1904	return nb_pkts;
1905}
1906
1907static uint16_t
1908i40e_xmit_pkts_simple(void *tx_queue,
1909		      struct rte_mbuf **tx_pkts,
1910		      uint16_t nb_pkts)
1911{
1912	uint16_t nb_tx = 0;
1913
1914	if (likely(nb_pkts <= I40E_TX_MAX_BURST))
1915		return tx_xmit_pkts((struct i40e_tx_queue *)tx_queue,
1916						tx_pkts, nb_pkts);
1917
1918	while (nb_pkts) {
1919		uint16_t ret, num = (uint16_t)RTE_MIN(nb_pkts,
1920						I40E_TX_MAX_BURST);
1921
1922		ret = tx_xmit_pkts((struct i40e_tx_queue *)tx_queue,
1923						&tx_pkts[nb_tx], num);
1924		nb_tx = (uint16_t)(nb_tx + ret);
1925		nb_pkts = (uint16_t)(nb_pkts - ret);
1926		if (ret < num)
1927			break;
1928	}
1929
1930	return nb_tx;
1931}
1932
1933/*
1934 * Find the VSI the queue belongs to. 'queue_idx' is the queue index
1935 * application used, which assume having sequential ones. But from driver's
1936 * perspective, it's different. For example, q0 belongs to FDIR VSI, q1-q64
1937 * to MAIN VSI, , q65-96 to SRIOV VSIs, q97-128 to VMDQ VSIs. For application
1938 * running on host, q1-64 and q97-128 can be used, total 96 queues. They can
1939 * use queue_idx from 0 to 95 to access queues, while real queue would be
1940 * different. This function will do a queue mapping to find VSI the queue
1941 * belongs to.
1942 */
1943static struct i40e_vsi*
1944i40e_pf_get_vsi_by_qindex(struct i40e_pf *pf, uint16_t queue_idx)
1945{
1946	/* the queue in MAIN VSI range */
1947	if (queue_idx < pf->main_vsi->nb_qps)
1948		return pf->main_vsi;
1949
1950	queue_idx -= pf->main_vsi->nb_qps;
1951
1952	/* queue_idx is greater than VMDQ VSIs range */
1953	if (queue_idx > pf->nb_cfg_vmdq_vsi * pf->vmdq_nb_qps - 1) {
1954		PMD_INIT_LOG(ERR, "queue_idx out of range. VMDQ configured?");
1955		return NULL;
1956	}
1957
1958	return pf->vmdq[queue_idx / pf->vmdq_nb_qps].vsi;
1959}
1960
1961static uint16_t
1962i40e_get_queue_offset_by_qindex(struct i40e_pf *pf, uint16_t queue_idx)
1963{
1964	/* the queue in MAIN VSI range */
1965	if (queue_idx < pf->main_vsi->nb_qps)
1966		return queue_idx;
1967
1968	/* It's VMDQ queues */
1969	queue_idx -= pf->main_vsi->nb_qps;
1970
1971	if (pf->nb_cfg_vmdq_vsi)
1972		return queue_idx % pf->vmdq_nb_qps;
1973	else {
1974		PMD_INIT_LOG(ERR, "Fail to get queue offset");
1975		return (uint16_t)(-1);
1976	}
1977}
1978
1979int
1980i40e_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1981{
1982	struct i40e_rx_queue *rxq;
1983	int err = -1;
1984	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1985
1986	PMD_INIT_FUNC_TRACE();
1987
1988	if (rx_queue_id < dev->data->nb_rx_queues) {
1989		rxq = dev->data->rx_queues[rx_queue_id];
1990
1991		err = i40e_alloc_rx_queue_mbufs(rxq);
1992		if (err) {
1993			PMD_DRV_LOG(ERR, "Failed to allocate RX queue mbuf");
1994			return err;
1995		}
1996
1997		rte_wmb();
1998
1999		/* Init the RX tail regieter. */
2000		I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
2001
2002		err = i40e_switch_rx_queue(hw, rxq->reg_idx, TRUE);
2003
2004		if (err) {
2005			PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on",
2006				    rx_queue_id);
2007
2008			i40e_rx_queue_release_mbufs(rxq);
2009			i40e_reset_rx_queue(rxq);
2010		} else
2011			dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
2012	}
2013
2014	return err;
2015}
2016
2017int
2018i40e_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
2019{
2020	struct i40e_rx_queue *rxq;
2021	int err;
2022	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2023
2024	if (rx_queue_id < dev->data->nb_rx_queues) {
2025		rxq = dev->data->rx_queues[rx_queue_id];
2026
2027		/*
2028		* rx_queue_id is queue id aplication refers to, while
2029		* rxq->reg_idx is the real queue index.
2030		*/
2031		err = i40e_switch_rx_queue(hw, rxq->reg_idx, FALSE);
2032
2033		if (err) {
2034			PMD_DRV_LOG(ERR, "Failed to switch RX queue %u off",
2035				    rx_queue_id);
2036			return err;
2037		}
2038		i40e_rx_queue_release_mbufs(rxq);
2039		i40e_reset_rx_queue(rxq);
2040		dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
2041	}
2042
2043	return 0;
2044}
2045
2046int
2047i40e_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
2048{
2049	int err = -1;
2050	struct i40e_tx_queue *txq;
2051	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2052
2053	PMD_INIT_FUNC_TRACE();
2054
2055	if (tx_queue_id < dev->data->nb_tx_queues) {
2056		txq = dev->data->tx_queues[tx_queue_id];
2057
2058		/*
2059		* tx_queue_id is queue id aplication refers to, while
2060		* rxq->reg_idx is the real queue index.
2061		*/
2062		err = i40e_switch_tx_queue(hw, txq->reg_idx, TRUE);
2063		if (err)
2064			PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on",
2065				    tx_queue_id);
2066		else
2067			dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
2068	}
2069
2070	return err;
2071}
2072
2073int
2074i40e_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
2075{
2076	struct i40e_tx_queue *txq;
2077	int err;
2078	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2079
2080	if (tx_queue_id < dev->data->nb_tx_queues) {
2081		txq = dev->data->tx_queues[tx_queue_id];
2082
2083		/*
2084		* tx_queue_id is queue id aplication refers to, while
2085		* txq->reg_idx is the real queue index.
2086		*/
2087		err = i40e_switch_tx_queue(hw, txq->reg_idx, FALSE);
2088
2089		if (err) {
2090			PMD_DRV_LOG(ERR, "Failed to switch TX queue %u of",
2091				    tx_queue_id);
2092			return err;
2093		}
2094
2095		i40e_tx_queue_release_mbufs(txq);
2096		i40e_reset_tx_queue(txq);
2097		dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
2098	}
2099
2100	return 0;
2101}
2102
2103const uint32_t *
2104i40e_dev_supported_ptypes_get(struct rte_eth_dev *dev)
2105{
2106	static const uint32_t ptypes[] = {
2107		/* refers to i40e_rxd_pkt_type_mapping() */
2108		RTE_PTYPE_L2_ETHER,
2109		RTE_PTYPE_L2_ETHER_TIMESYNC,
2110		RTE_PTYPE_L2_ETHER_LLDP,
2111		RTE_PTYPE_L2_ETHER_ARP,
2112		RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
2113		RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
2114		RTE_PTYPE_L4_FRAG,
2115		RTE_PTYPE_L4_ICMP,
2116		RTE_PTYPE_L4_NONFRAG,
2117		RTE_PTYPE_L4_SCTP,
2118		RTE_PTYPE_L4_TCP,
2119		RTE_PTYPE_L4_UDP,
2120		RTE_PTYPE_TUNNEL_GRENAT,
2121		RTE_PTYPE_TUNNEL_IP,
2122		RTE_PTYPE_INNER_L2_ETHER,
2123		RTE_PTYPE_INNER_L2_ETHER_VLAN,
2124		RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN,
2125		RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN,
2126		RTE_PTYPE_INNER_L4_FRAG,
2127		RTE_PTYPE_INNER_L4_ICMP,
2128		RTE_PTYPE_INNER_L4_NONFRAG,
2129		RTE_PTYPE_INNER_L4_SCTP,
2130		RTE_PTYPE_INNER_L4_TCP,
2131		RTE_PTYPE_INNER_L4_UDP,
2132		RTE_PTYPE_UNKNOWN
2133	};
2134
2135	if (dev->rx_pkt_burst == i40e_recv_pkts ||
2136#ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC
2137	    dev->rx_pkt_burst == i40e_recv_pkts_bulk_alloc ||
2138#endif
2139	    dev->rx_pkt_burst == i40e_recv_scattered_pkts)
2140		return ptypes;
2141	return NULL;
2142}
2143
2144int
2145i40e_dev_rx_queue_setup(struct rte_eth_dev *dev,
2146			uint16_t queue_idx,
2147			uint16_t nb_desc,
2148			unsigned int socket_id,
2149			const struct rte_eth_rxconf *rx_conf,
2150			struct rte_mempool *mp)
2151{
2152	struct i40e_vsi *vsi;
2153	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2154	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2155	struct i40e_adapter *ad =
2156		I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
2157	struct i40e_rx_queue *rxq;
2158	const struct rte_memzone *rz;
2159	uint32_t ring_size;
2160	uint16_t len, i;
2161	uint16_t base, bsf, tc_mapping;
2162	int use_def_burst_func = 1;
2163
2164	if (hw->mac.type == I40E_MAC_VF || hw->mac.type == I40E_MAC_X722_VF) {
2165		struct i40e_vf *vf =
2166			I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2167		vsi = &vf->vsi;
2168	} else
2169		vsi = i40e_pf_get_vsi_by_qindex(pf, queue_idx);
2170
2171	if (vsi == NULL) {
2172		PMD_DRV_LOG(ERR, "VSI not available or queue "
2173			    "index exceeds the maximum");
2174		return I40E_ERR_PARAM;
2175	}
2176	if (nb_desc % I40E_ALIGN_RING_DESC != 0 ||
2177			(nb_desc > I40E_MAX_RING_DESC) ||
2178			(nb_desc < I40E_MIN_RING_DESC)) {
2179		PMD_DRV_LOG(ERR, "Number (%u) of receive descriptors is "
2180			    "invalid", nb_desc);
2181		return I40E_ERR_PARAM;
2182	}
2183
2184	/* Free memory if needed */
2185	if (dev->data->rx_queues[queue_idx]) {
2186		i40e_dev_rx_queue_release(dev->data->rx_queues[queue_idx]);
2187		dev->data->rx_queues[queue_idx] = NULL;
2188	}
2189
2190	/* Allocate the rx queue data structure */
2191	rxq = rte_zmalloc_socket("i40e rx queue",
2192				 sizeof(struct i40e_rx_queue),
2193				 RTE_CACHE_LINE_SIZE,
2194				 socket_id);
2195	if (!rxq) {
2196		PMD_DRV_LOG(ERR, "Failed to allocate memory for "
2197			    "rx queue data structure");
2198		return -ENOMEM;
2199	}
2200	rxq->mp = mp;
2201	rxq->nb_rx_desc = nb_desc;
2202	rxq->rx_free_thresh = rx_conf->rx_free_thresh;
2203	rxq->queue_id = queue_idx;
2204	if (hw->mac.type == I40E_MAC_VF || hw->mac.type == I40E_MAC_X722_VF)
2205		rxq->reg_idx = queue_idx;
2206	else /* PF device */
2207		rxq->reg_idx = vsi->base_queue +
2208			i40e_get_queue_offset_by_qindex(pf, queue_idx);
2209
2210	rxq->port_id = dev->data->port_id;
2211	rxq->crc_len = (uint8_t) ((dev->data->dev_conf.rxmode.hw_strip_crc) ?
2212							0 : ETHER_CRC_LEN);
2213	rxq->drop_en = rx_conf->rx_drop_en;
2214	rxq->vsi = vsi;
2215	rxq->rx_deferred_start = rx_conf->rx_deferred_start;
2216
2217	/* Allocate the maximun number of RX ring hardware descriptor. */
2218	ring_size = sizeof(union i40e_rx_desc) * I40E_MAX_RING_DESC;
2219	ring_size = RTE_ALIGN(ring_size, I40E_DMA_MEM_ALIGN);
2220	rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx,
2221			      ring_size, I40E_RING_BASE_ALIGN, socket_id);
2222	if (!rz) {
2223		i40e_dev_rx_queue_release(rxq);
2224		PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for RX");
2225		return -ENOMEM;
2226	}
2227
2228	/* Zero all the descriptors in the ring. */
2229	memset(rz->addr, 0, ring_size);
2230
2231	rxq->rx_ring_phys_addr = rte_mem_phy2mch(rz->memseg_id, rz->phys_addr);
2232	rxq->rx_ring = (union i40e_rx_desc *)rz->addr;
2233
2234#ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC
2235	len = (uint16_t)(nb_desc + RTE_PMD_I40E_RX_MAX_BURST);
2236#else
2237	len = nb_desc;
2238#endif
2239
2240	/* Allocate the software ring. */
2241	rxq->sw_ring =
2242		rte_zmalloc_socket("i40e rx sw ring",
2243				   sizeof(struct i40e_rx_entry) * len,
2244				   RTE_CACHE_LINE_SIZE,
2245				   socket_id);
2246	if (!rxq->sw_ring) {
2247		i40e_dev_rx_queue_release(rxq);
2248		PMD_DRV_LOG(ERR, "Failed to allocate memory for SW ring");
2249		return -ENOMEM;
2250	}
2251
2252	i40e_reset_rx_queue(rxq);
2253	rxq->q_set = TRUE;
2254	dev->data->rx_queues[queue_idx] = rxq;
2255
2256	use_def_burst_func = check_rx_burst_bulk_alloc_preconditions(rxq);
2257
2258	if (!use_def_burst_func) {
2259#ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC
2260		PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
2261			     "satisfied. Rx Burst Bulk Alloc function will be "
2262			     "used on port=%d, queue=%d.",
2263			     rxq->port_id, rxq->queue_id);
2264#endif /* RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC */
2265	} else {
2266		PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
2267			     "not satisfied, Scattered Rx is requested, "
2268			     "or RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC is "
2269			     "not enabled on port=%d, queue=%d.",
2270			     rxq->port_id, rxq->queue_id);
2271		ad->rx_bulk_alloc_allowed = false;
2272	}
2273
2274	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
2275		if (!(vsi->enabled_tc & (1 << i)))
2276			continue;
2277		tc_mapping = rte_le_to_cpu_16(vsi->info.tc_mapping[i]);
2278		base = (tc_mapping & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) >>
2279			I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT;
2280		bsf = (tc_mapping & I40E_AQ_VSI_TC_QUE_NUMBER_MASK) >>
2281			I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT;
2282
2283		if (queue_idx >= base && queue_idx < (base + BIT(bsf)))
2284			rxq->dcb_tc = i;
2285	}
2286
2287	return 0;
2288}
2289
2290void
2291i40e_dev_rx_queue_release(void *rxq)
2292{
2293	struct i40e_rx_queue *q = (struct i40e_rx_queue *)rxq;
2294
2295	if (!q) {
2296		PMD_DRV_LOG(DEBUG, "Pointer to rxq is NULL");
2297		return;
2298	}
2299
2300	i40e_rx_queue_release_mbufs(q);
2301	rte_free(q->sw_ring);
2302	rte_free(q);
2303}
2304
2305uint32_t
2306i40e_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id)
2307{
2308#define I40E_RXQ_SCAN_INTERVAL 4
2309	volatile union i40e_rx_desc *rxdp;
2310	struct i40e_rx_queue *rxq;
2311	uint16_t desc = 0;
2312
2313	if (unlikely(rx_queue_id >= dev->data->nb_rx_queues)) {
2314		PMD_DRV_LOG(ERR, "Invalid RX queue id %u", rx_queue_id);
2315		return 0;
2316	}
2317
2318	rxq = dev->data->rx_queues[rx_queue_id];
2319	rxdp = &(rxq->rx_ring[rxq->rx_tail]);
2320	while ((desc < rxq->nb_rx_desc) &&
2321		((rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len) &
2322		I40E_RXD_QW1_STATUS_MASK) >> I40E_RXD_QW1_STATUS_SHIFT) &
2323				(1 << I40E_RX_DESC_STATUS_DD_SHIFT)) {
2324		/**
2325		 * Check the DD bit of a rx descriptor of each 4 in a group,
2326		 * to avoid checking too frequently and downgrading performance
2327		 * too much.
2328		 */
2329		desc += I40E_RXQ_SCAN_INTERVAL;
2330		rxdp += I40E_RXQ_SCAN_INTERVAL;
2331		if (rxq->rx_tail + desc >= rxq->nb_rx_desc)
2332			rxdp = &(rxq->rx_ring[rxq->rx_tail +
2333					desc - rxq->nb_rx_desc]);
2334	}
2335
2336	return desc;
2337}
2338
2339int
2340i40e_dev_rx_descriptor_done(void *rx_queue, uint16_t offset)
2341{
2342	volatile union i40e_rx_desc *rxdp;
2343	struct i40e_rx_queue *rxq = rx_queue;
2344	uint16_t desc;
2345	int ret;
2346
2347	if (unlikely(offset >= rxq->nb_rx_desc)) {
2348		PMD_DRV_LOG(ERR, "Invalid RX queue id %u", offset);
2349		return 0;
2350	}
2351
2352	desc = rxq->rx_tail + offset;
2353	if (desc >= rxq->nb_rx_desc)
2354		desc -= rxq->nb_rx_desc;
2355
2356	rxdp = &(rxq->rx_ring[desc]);
2357
2358	ret = !!(((rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len) &
2359		I40E_RXD_QW1_STATUS_MASK) >> I40E_RXD_QW1_STATUS_SHIFT) &
2360				(1 << I40E_RX_DESC_STATUS_DD_SHIFT));
2361
2362	return ret;
2363}
2364
2365int
2366i40e_dev_tx_queue_setup(struct rte_eth_dev *dev,
2367			uint16_t queue_idx,
2368			uint16_t nb_desc,
2369			unsigned int socket_id,
2370			const struct rte_eth_txconf *tx_conf)
2371{
2372	struct i40e_vsi *vsi;
2373	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2374	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
2375	struct i40e_tx_queue *txq;
2376	const struct rte_memzone *tz;
2377	uint32_t ring_size;
2378	uint16_t tx_rs_thresh, tx_free_thresh;
2379	uint16_t i, base, bsf, tc_mapping;
2380
2381	if (hw->mac.type == I40E_MAC_VF || hw->mac.type == I40E_MAC_X722_VF) {
2382		struct i40e_vf *vf =
2383			I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2384		vsi = &vf->vsi;
2385	} else
2386		vsi = i40e_pf_get_vsi_by_qindex(pf, queue_idx);
2387
2388	if (vsi == NULL) {
2389		PMD_DRV_LOG(ERR, "VSI is NULL, or queue index (%u) "
2390			    "exceeds the maximum", queue_idx);
2391		return I40E_ERR_PARAM;
2392	}
2393
2394	if (nb_desc % I40E_ALIGN_RING_DESC != 0 ||
2395			(nb_desc > I40E_MAX_RING_DESC) ||
2396			(nb_desc < I40E_MIN_RING_DESC)) {
2397		PMD_DRV_LOG(ERR, "Number (%u) of transmit descriptors is "
2398			    "invalid", nb_desc);
2399		return I40E_ERR_PARAM;
2400	}
2401
2402	/**
2403	 * The following two parameters control the setting of the RS bit on
2404	 * transmit descriptors. TX descriptors will have their RS bit set
2405	 * after txq->tx_rs_thresh descriptors have been used. The TX
2406	 * descriptor ring will be cleaned after txq->tx_free_thresh
2407	 * descriptors are used or if the number of descriptors required to
2408	 * transmit a packet is greater than the number of free TX descriptors.
2409	 *
2410	 * The following constraints must be satisfied:
2411	 *  - tx_rs_thresh must be greater than 0.
2412	 *  - tx_rs_thresh must be less than the size of the ring minus 2.
2413	 *  - tx_rs_thresh must be less than or equal to tx_free_thresh.
2414	 *  - tx_rs_thresh must be a divisor of the ring size.
2415	 *  - tx_free_thresh must be greater than 0.
2416	 *  - tx_free_thresh must be less than the size of the ring minus 3.
2417	 *
2418	 * One descriptor in the TX ring is used as a sentinel to avoid a H/W
2419	 * race condition, hence the maximum threshold constraints. When set
2420	 * to zero use default values.
2421	 */
2422	tx_rs_thresh = (uint16_t)((tx_conf->tx_rs_thresh) ?
2423		tx_conf->tx_rs_thresh : DEFAULT_TX_RS_THRESH);
2424	tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ?
2425		tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH);
2426	if (tx_rs_thresh >= (nb_desc - 2)) {
2427		PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than the "
2428			     "number of TX descriptors minus 2. "
2429			     "(tx_rs_thresh=%u port=%d queue=%d)",
2430			     (unsigned int)tx_rs_thresh,
2431			     (int)dev->data->port_id,
2432			     (int)queue_idx);
2433		return I40E_ERR_PARAM;
2434	}
2435	if (tx_free_thresh >= (nb_desc - 3)) {
2436		PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than the "
2437			     "tx_free_thresh must be less than the "
2438			     "number of TX descriptors minus 3. "
2439			     "(tx_free_thresh=%u port=%d queue=%d)",
2440			     (unsigned int)tx_free_thresh,
2441			     (int)dev->data->port_id,
2442			     (int)queue_idx);
2443		return I40E_ERR_PARAM;
2444	}
2445	if (tx_rs_thresh > tx_free_thresh) {
2446		PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than or "
2447			     "equal to tx_free_thresh. (tx_free_thresh=%u"
2448			     " tx_rs_thresh=%u port=%d queue=%d)",
2449			     (unsigned int)tx_free_thresh,
2450			     (unsigned int)tx_rs_thresh,
2451			     (int)dev->data->port_id,
2452			     (int)queue_idx);
2453		return I40E_ERR_PARAM;
2454	}
2455	if ((nb_desc % tx_rs_thresh) != 0) {
2456		PMD_INIT_LOG(ERR, "tx_rs_thresh must be a divisor of the "
2457			     "number of TX descriptors. (tx_rs_thresh=%u"
2458			     " port=%d queue=%d)",
2459			     (unsigned int)tx_rs_thresh,
2460			     (int)dev->data->port_id,
2461			     (int)queue_idx);
2462		return I40E_ERR_PARAM;
2463	}
2464	if ((tx_rs_thresh > 1) && (tx_conf->tx_thresh.wthresh != 0)) {
2465		PMD_INIT_LOG(ERR, "TX WTHRESH must be set to 0 if "
2466			     "tx_rs_thresh is greater than 1. "
2467			     "(tx_rs_thresh=%u port=%d queue=%d)",
2468			     (unsigned int)tx_rs_thresh,
2469			     (int)dev->data->port_id,
2470			     (int)queue_idx);
2471		return I40E_ERR_PARAM;
2472	}
2473
2474	/* Free memory if needed. */
2475	if (dev->data->tx_queues[queue_idx]) {
2476		i40e_dev_tx_queue_release(dev->data->tx_queues[queue_idx]);
2477		dev->data->tx_queues[queue_idx] = NULL;
2478	}
2479
2480	/* Allocate the TX queue data structure. */
2481	txq = rte_zmalloc_socket("i40e tx queue",
2482				  sizeof(struct i40e_tx_queue),
2483				  RTE_CACHE_LINE_SIZE,
2484				  socket_id);
2485	if (!txq) {
2486		PMD_DRV_LOG(ERR, "Failed to allocate memory for "
2487			    "tx queue structure");
2488		return -ENOMEM;
2489	}
2490
2491	/* Allocate TX hardware ring descriptors. */
2492	ring_size = sizeof(struct i40e_tx_desc) * I40E_MAX_RING_DESC;
2493	ring_size = RTE_ALIGN(ring_size, I40E_DMA_MEM_ALIGN);
2494	tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx,
2495			      ring_size, I40E_RING_BASE_ALIGN, socket_id);
2496	if (!tz) {
2497		i40e_dev_tx_queue_release(txq);
2498		PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for TX");
2499		return -ENOMEM;
2500	}
2501
2502	txq->nb_tx_desc = nb_desc;
2503	txq->tx_rs_thresh = tx_rs_thresh;
2504	txq->tx_free_thresh = tx_free_thresh;
2505	txq->pthresh = tx_conf->tx_thresh.pthresh;
2506	txq->hthresh = tx_conf->tx_thresh.hthresh;
2507	txq->wthresh = tx_conf->tx_thresh.wthresh;
2508	txq->queue_id = queue_idx;
2509	if (hw->mac.type == I40E_MAC_VF || hw->mac.type == I40E_MAC_X722_VF)
2510		txq->reg_idx = queue_idx;
2511	else /* PF device */
2512		txq->reg_idx = vsi->base_queue +
2513			i40e_get_queue_offset_by_qindex(pf, queue_idx);
2514
2515	txq->port_id = dev->data->port_id;
2516	txq->txq_flags = tx_conf->txq_flags;
2517	txq->vsi = vsi;
2518	txq->tx_deferred_start = tx_conf->tx_deferred_start;
2519
2520	txq->tx_ring_phys_addr = rte_mem_phy2mch(tz->memseg_id, tz->phys_addr);
2521	txq->tx_ring = (struct i40e_tx_desc *)tz->addr;
2522
2523	/* Allocate software ring */
2524	txq->sw_ring =
2525		rte_zmalloc_socket("i40e tx sw ring",
2526				   sizeof(struct i40e_tx_entry) * nb_desc,
2527				   RTE_CACHE_LINE_SIZE,
2528				   socket_id);
2529	if (!txq->sw_ring) {
2530		i40e_dev_tx_queue_release(txq);
2531		PMD_DRV_LOG(ERR, "Failed to allocate memory for SW TX ring");
2532		return -ENOMEM;
2533	}
2534
2535	i40e_reset_tx_queue(txq);
2536	txq->q_set = TRUE;
2537	dev->data->tx_queues[queue_idx] = txq;
2538
2539	/* Use a simple TX queue without offloads or multi segs if possible */
2540	i40e_set_tx_function_flag(dev, txq);
2541
2542	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
2543		if (!(vsi->enabled_tc & (1 << i)))
2544			continue;
2545		tc_mapping = rte_le_to_cpu_16(vsi->info.tc_mapping[i]);
2546		base = (tc_mapping & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) >>
2547			I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT;
2548		bsf = (tc_mapping & I40E_AQ_VSI_TC_QUE_NUMBER_MASK) >>
2549			I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT;
2550
2551		if (queue_idx >= base && queue_idx < (base + BIT(bsf)))
2552			txq->dcb_tc = i;
2553	}
2554
2555	return 0;
2556}
2557
2558void
2559i40e_dev_tx_queue_release(void *txq)
2560{
2561	struct i40e_tx_queue *q = (struct i40e_tx_queue *)txq;
2562
2563	if (!q) {
2564		PMD_DRV_LOG(DEBUG, "Pointer to TX queue is NULL");
2565		return;
2566	}
2567
2568	i40e_tx_queue_release_mbufs(q);
2569	rte_free(q->sw_ring);
2570	rte_free(q);
2571}
2572
2573const struct rte_memzone *
2574i40e_memzone_reserve(const char *name, uint32_t len, int socket_id)
2575{
2576	const struct rte_memzone *mz;
2577
2578	mz = rte_memzone_lookup(name);
2579	if (mz)
2580		return mz;
2581
2582	if (rte_xen_dom0_supported())
2583		mz = rte_memzone_reserve_bounded(name, len,
2584				socket_id, 0, I40E_RING_BASE_ALIGN, RTE_PGSIZE_2M);
2585	else
2586		mz = rte_memzone_reserve_aligned(name, len,
2587				socket_id, 0, I40E_RING_BASE_ALIGN);
2588	return mz;
2589}
2590
2591void
2592i40e_rx_queue_release_mbufs(struct i40e_rx_queue *rxq)
2593{
2594	uint16_t i;
2595
2596	/* SSE Vector driver has a different way of releasing mbufs. */
2597	if (rxq->rx_using_sse) {
2598		i40e_rx_queue_release_mbufs_vec(rxq);
2599		return;
2600	}
2601
2602	if (!rxq->sw_ring) {
2603		PMD_DRV_LOG(DEBUG, "Pointer to sw_ring is NULL");
2604		return;
2605	}
2606
2607	for (i = 0; i < rxq->nb_rx_desc; i++) {
2608		if (rxq->sw_ring[i].mbuf) {
2609			rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
2610			rxq->sw_ring[i].mbuf = NULL;
2611		}
2612	}
2613#ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC
2614	if (rxq->rx_nb_avail == 0)
2615		return;
2616	for (i = 0; i < rxq->rx_nb_avail; i++) {
2617		struct rte_mbuf *mbuf;
2618
2619		mbuf = rxq->rx_stage[rxq->rx_next_avail + i];
2620		rte_pktmbuf_free_seg(mbuf);
2621	}
2622	rxq->rx_nb_avail = 0;
2623#endif /* RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC */
2624}
2625
2626void
2627i40e_reset_rx_queue(struct i40e_rx_queue *rxq)
2628{
2629	unsigned i;
2630	uint16_t len;
2631
2632	if (!rxq) {
2633		PMD_DRV_LOG(DEBUG, "Pointer to rxq is NULL");
2634		return;
2635	}
2636
2637#ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC
2638	if (check_rx_burst_bulk_alloc_preconditions(rxq) == 0)
2639		len = (uint16_t)(rxq->nb_rx_desc + RTE_PMD_I40E_RX_MAX_BURST);
2640	else
2641#endif /* RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC */
2642		len = rxq->nb_rx_desc;
2643
2644	for (i = 0; i < len * sizeof(union i40e_rx_desc); i++)
2645		((volatile char *)rxq->rx_ring)[i] = 0;
2646
2647#ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC
2648	memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf));
2649	for (i = 0; i < RTE_PMD_I40E_RX_MAX_BURST; ++i)
2650		rxq->sw_ring[rxq->nb_rx_desc + i].mbuf = &rxq->fake_mbuf;
2651
2652	rxq->rx_nb_avail = 0;
2653	rxq->rx_next_avail = 0;
2654	rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1);
2655#endif /* RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC */
2656	rxq->rx_tail = 0;
2657	rxq->nb_rx_hold = 0;
2658	rxq->pkt_first_seg = NULL;
2659	rxq->pkt_last_seg = NULL;
2660
2661	rxq->rxrearm_start = 0;
2662	rxq->rxrearm_nb = 0;
2663}
2664
2665void
2666i40e_tx_queue_release_mbufs(struct i40e_tx_queue *txq)
2667{
2668	uint16_t i;
2669
2670	if (!txq || !txq->sw_ring) {
2671		PMD_DRV_LOG(DEBUG, "Pointer to rxq or sw_ring is NULL");
2672		return;
2673	}
2674
2675	for (i = 0; i < txq->nb_tx_desc; i++) {
2676		if (txq->sw_ring[i].mbuf) {
2677			rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
2678			txq->sw_ring[i].mbuf = NULL;
2679		}
2680	}
2681}
2682
2683void
2684i40e_reset_tx_queue(struct i40e_tx_queue *txq)
2685{
2686	struct i40e_tx_entry *txe;
2687	uint16_t i, prev, size;
2688
2689	if (!txq) {
2690		PMD_DRV_LOG(DEBUG, "Pointer to txq is NULL");
2691		return;
2692	}
2693
2694	txe = txq->sw_ring;
2695	size = sizeof(struct i40e_tx_desc) * txq->nb_tx_desc;
2696	for (i = 0; i < size; i++)
2697		((volatile char *)txq->tx_ring)[i] = 0;
2698
2699	prev = (uint16_t)(txq->nb_tx_desc - 1);
2700	for (i = 0; i < txq->nb_tx_desc; i++) {
2701		volatile struct i40e_tx_desc *txd = &txq->tx_ring[i];
2702
2703		txd->cmd_type_offset_bsz =
2704			rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE);
2705		txe[i].mbuf =  NULL;
2706		txe[i].last_id = i;
2707		txe[prev].next_id = i;
2708		prev = i;
2709	}
2710
2711	txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
2712	txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
2713
2714	txq->tx_tail = 0;
2715	txq->nb_tx_used = 0;
2716
2717	txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1);
2718	txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1);
2719}
2720
2721/* Init the TX queue in hardware */
2722int
2723i40e_tx_queue_init(struct i40e_tx_queue *txq)
2724{
2725	enum i40e_status_code err = I40E_SUCCESS;
2726	struct i40e_vsi *vsi = txq->vsi;
2727	struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2728	uint16_t pf_q = txq->reg_idx;
2729	struct i40e_hmc_obj_txq tx_ctx;
2730	uint32_t qtx_ctl;
2731
2732	/* clear the context structure first */
2733	memset(&tx_ctx, 0, sizeof(tx_ctx));
2734	tx_ctx.new_context = 1;
2735	tx_ctx.base = txq->tx_ring_phys_addr / I40E_QUEUE_BASE_ADDR_UNIT;
2736	tx_ctx.qlen = txq->nb_tx_desc;
2737
2738#ifdef RTE_LIBRTE_IEEE1588
2739	tx_ctx.timesync_ena = 1;
2740#endif
2741	tx_ctx.rdylist = rte_le_to_cpu_16(vsi->info.qs_handle[txq->dcb_tc]);
2742	if (vsi->type == I40E_VSI_FDIR)
2743		tx_ctx.fd_ena = TRUE;
2744
2745	err = i40e_clear_lan_tx_queue_context(hw, pf_q);
2746	if (err != I40E_SUCCESS) {
2747		PMD_DRV_LOG(ERR, "Failure of clean lan tx queue context");
2748		return err;
2749	}
2750
2751	err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
2752	if (err != I40E_SUCCESS) {
2753		PMD_DRV_LOG(ERR, "Failure of set lan tx queue context");
2754		return err;
2755	}
2756
2757	/* Now associate this queue with this PCI function */
2758	qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
2759	qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
2760					I40E_QTX_CTL_PF_INDX_MASK);
2761	I40E_WRITE_REG(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
2762	I40E_WRITE_FLUSH(hw);
2763
2764	txq->qtx_tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
2765
2766	return err;
2767}
2768
2769int
2770i40e_alloc_rx_queue_mbufs(struct i40e_rx_queue *rxq)
2771{
2772	struct i40e_rx_entry *rxe = rxq->sw_ring;
2773	uint64_t dma_addr;
2774	uint16_t i;
2775
2776	for (i = 0; i < rxq->nb_rx_desc; i++) {
2777		volatile union i40e_rx_desc *rxd;
2778		struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mp);
2779
2780		if (unlikely(!mbuf)) {
2781			PMD_DRV_LOG(ERR, "Failed to allocate mbuf for RX");
2782			return -ENOMEM;
2783		}
2784
2785		rte_mbuf_refcnt_set(mbuf, 1);
2786		mbuf->next = NULL;
2787		mbuf->data_off = RTE_PKTMBUF_HEADROOM;
2788		mbuf->nb_segs = 1;
2789		mbuf->port = rxq->port_id;
2790
2791		dma_addr =
2792			rte_cpu_to_le_64(rte_mbuf_data_dma_addr_default(mbuf));
2793
2794		rxd = &rxq->rx_ring[i];
2795		rxd->read.pkt_addr = dma_addr;
2796		rxd->read.hdr_addr = 0;
2797#ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC
2798		rxd->read.rsvd1 = 0;
2799		rxd->read.rsvd2 = 0;
2800#endif /* RTE_LIBRTE_I40E_16BYTE_RX_DESC */
2801
2802		rxe[i].mbuf = mbuf;
2803	}
2804
2805	return 0;
2806}
2807
2808/*
2809 * Calculate the buffer length, and check the jumbo frame
2810 * and maximum packet length.
2811 */
2812static int
2813i40e_rx_queue_config(struct i40e_rx_queue *rxq)
2814{
2815	struct i40e_pf *pf = I40E_VSI_TO_PF(rxq->vsi);
2816	struct i40e_hw *hw = I40E_VSI_TO_HW(rxq->vsi);
2817	struct rte_eth_dev_data *data = pf->dev_data;
2818	uint16_t buf_size, len;
2819
2820	buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
2821		RTE_PKTMBUF_HEADROOM);
2822
2823	switch (pf->flags & (I40E_FLAG_HEADER_SPLIT_DISABLED |
2824			I40E_FLAG_HEADER_SPLIT_ENABLED)) {
2825	case I40E_FLAG_HEADER_SPLIT_ENABLED: /* Not supported */
2826		rxq->rx_hdr_len = RTE_ALIGN(I40E_RXBUF_SZ_1024,
2827				(1 << I40E_RXQ_CTX_HBUFF_SHIFT));
2828		rxq->rx_buf_len = RTE_ALIGN(I40E_RXBUF_SZ_2048,
2829				(1 << I40E_RXQ_CTX_DBUFF_SHIFT));
2830		rxq->hs_mode = i40e_header_split_enabled;
2831		break;
2832	case I40E_FLAG_HEADER_SPLIT_DISABLED:
2833	default:
2834		rxq->rx_hdr_len = 0;
2835		rxq->rx_buf_len = RTE_ALIGN(buf_size,
2836			(1 << I40E_RXQ_CTX_DBUFF_SHIFT));
2837		rxq->hs_mode = i40e_header_split_none;
2838		break;
2839	}
2840
2841	len = hw->func_caps.rx_buf_chain_len * rxq->rx_buf_len;
2842	rxq->max_pkt_len = RTE_MIN(len, data->dev_conf.rxmode.max_rx_pkt_len);
2843	if (data->dev_conf.rxmode.jumbo_frame == 1) {
2844		if (rxq->max_pkt_len <= ETHER_MAX_LEN ||
2845			rxq->max_pkt_len > I40E_FRAME_SIZE_MAX) {
2846			PMD_DRV_LOG(ERR, "maximum packet length must "
2847				    "be larger than %u and smaller than %u,"
2848				    "as jumbo frame is enabled",
2849				    (uint32_t)ETHER_MAX_LEN,
2850				    (uint32_t)I40E_FRAME_SIZE_MAX);
2851			return I40E_ERR_CONFIG;
2852		}
2853	} else {
2854		if (rxq->max_pkt_len < ETHER_MIN_LEN ||
2855			rxq->max_pkt_len > ETHER_MAX_LEN) {
2856			PMD_DRV_LOG(ERR, "maximum packet length must be "
2857				    "larger than %u and smaller than %u, "
2858				    "as jumbo frame is disabled",
2859				    (uint32_t)ETHER_MIN_LEN,
2860				    (uint32_t)ETHER_MAX_LEN);
2861			return I40E_ERR_CONFIG;
2862		}
2863	}
2864
2865	return 0;
2866}
2867
2868/* Init the RX queue in hardware */
2869int
2870i40e_rx_queue_init(struct i40e_rx_queue *rxq)
2871{
2872	int err = I40E_SUCCESS;
2873	struct i40e_hw *hw = I40E_VSI_TO_HW(rxq->vsi);
2874	struct rte_eth_dev_data *dev_data = I40E_VSI_TO_DEV_DATA(rxq->vsi);
2875	uint16_t pf_q = rxq->reg_idx;
2876	uint16_t buf_size;
2877	struct i40e_hmc_obj_rxq rx_ctx;
2878
2879	err = i40e_rx_queue_config(rxq);
2880	if (err < 0) {
2881		PMD_DRV_LOG(ERR, "Failed to config RX queue");
2882		return err;
2883	}
2884
2885	/* Clear the context structure first */
2886	memset(&rx_ctx, 0, sizeof(struct i40e_hmc_obj_rxq));
2887	rx_ctx.dbuff = rxq->rx_buf_len >> I40E_RXQ_CTX_DBUFF_SHIFT;
2888	rx_ctx.hbuff = rxq->rx_hdr_len >> I40E_RXQ_CTX_HBUFF_SHIFT;
2889
2890	rx_ctx.base = rxq->rx_ring_phys_addr / I40E_QUEUE_BASE_ADDR_UNIT;
2891	rx_ctx.qlen = rxq->nb_rx_desc;
2892#ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC
2893	rx_ctx.dsize = 1;
2894#endif
2895	rx_ctx.dtype = rxq->hs_mode;
2896	if (rxq->hs_mode)
2897		rx_ctx.hsplit_0 = I40E_HEADER_SPLIT_ALL;
2898	else
2899		rx_ctx.hsplit_0 = I40E_HEADER_SPLIT_NONE;
2900	rx_ctx.rxmax = rxq->max_pkt_len;
2901	rx_ctx.tphrdesc_ena = 1;
2902	rx_ctx.tphwdesc_ena = 1;
2903	rx_ctx.tphdata_ena = 1;
2904	rx_ctx.tphhead_ena = 1;
2905	rx_ctx.lrxqthresh = 2;
2906	rx_ctx.crcstrip = (rxq->crc_len == 0) ? 1 : 0;
2907	rx_ctx.l2tsel = 1;
2908	/* showiv indicates if inner VLAN is stripped inside of tunnel
2909	 * packet. When set it to 1, vlan information is stripped from
2910	 * the inner header, but the hardware does not put it in the
2911	 * descriptor. So set it zero by default.
2912	 */
2913	rx_ctx.showiv = 0;
2914	rx_ctx.prefena = 1;
2915
2916	err = i40e_clear_lan_rx_queue_context(hw, pf_q);
2917	if (err != I40E_SUCCESS) {
2918		PMD_DRV_LOG(ERR, "Failed to clear LAN RX queue context");
2919		return err;
2920	}
2921	err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
2922	if (err != I40E_SUCCESS) {
2923		PMD_DRV_LOG(ERR, "Failed to set LAN RX queue context");
2924		return err;
2925	}
2926
2927	rxq->qrx_tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
2928
2929	buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
2930		RTE_PKTMBUF_HEADROOM);
2931
2932	/* Check if scattered RX needs to be used. */
2933	if ((rxq->max_pkt_len + 2 * I40E_VLAN_TAG_SIZE) > buf_size) {
2934		dev_data->scattered_rx = 1;
2935	}
2936
2937	/* Init the RX tail regieter. */
2938	I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
2939
2940	return 0;
2941}
2942
2943void
2944i40e_dev_clear_queues(struct rte_eth_dev *dev)
2945{
2946	uint16_t i;
2947
2948	PMD_INIT_FUNC_TRACE();
2949
2950	for (i = 0; i < dev->data->nb_tx_queues; i++) {
2951		if (!dev->data->tx_queues[i])
2952			continue;
2953		i40e_tx_queue_release_mbufs(dev->data->tx_queues[i]);
2954		i40e_reset_tx_queue(dev->data->tx_queues[i]);
2955	}
2956
2957	for (i = 0; i < dev->data->nb_rx_queues; i++) {
2958		if (!dev->data->rx_queues[i])
2959			continue;
2960		i40e_rx_queue_release_mbufs(dev->data->rx_queues[i]);
2961		i40e_reset_rx_queue(dev->data->rx_queues[i]);
2962	}
2963}
2964
2965void
2966i40e_dev_free_queues(struct rte_eth_dev *dev)
2967{
2968	uint16_t i;
2969
2970	PMD_INIT_FUNC_TRACE();
2971
2972	for (i = 0; i < dev->data->nb_rx_queues; i++) {
2973		if (!dev->data->rx_queues[i])
2974			continue;
2975		i40e_dev_rx_queue_release(dev->data->rx_queues[i]);
2976		dev->data->rx_queues[i] = NULL;
2977	}
2978	dev->data->nb_rx_queues = 0;
2979
2980	for (i = 0; i < dev->data->nb_tx_queues; i++) {
2981		if (!dev->data->tx_queues[i])
2982			continue;
2983		i40e_dev_tx_queue_release(dev->data->tx_queues[i]);
2984		dev->data->tx_queues[i] = NULL;
2985	}
2986	dev->data->nb_tx_queues = 0;
2987}
2988
2989#define I40E_FDIR_NUM_TX_DESC  I40E_MIN_RING_DESC
2990#define I40E_FDIR_NUM_RX_DESC  I40E_MIN_RING_DESC
2991
2992enum i40e_status_code
2993i40e_fdir_setup_tx_resources(struct i40e_pf *pf)
2994{
2995	struct i40e_tx_queue *txq;
2996	const struct rte_memzone *tz = NULL;
2997	uint32_t ring_size;
2998	struct rte_eth_dev *dev;
2999
3000	if (!pf) {
3001		PMD_DRV_LOG(ERR, "PF is not available");
3002		return I40E_ERR_BAD_PTR;
3003	}
3004
3005	dev = pf->adapter->eth_dev;
3006
3007	/* Allocate the TX queue data structure. */
3008	txq = rte_zmalloc_socket("i40e fdir tx queue",
3009				  sizeof(struct i40e_tx_queue),
3010				  RTE_CACHE_LINE_SIZE,
3011				  SOCKET_ID_ANY);
3012	if (!txq) {
3013		PMD_DRV_LOG(ERR, "Failed to allocate memory for "
3014					"tx queue structure.");
3015		return I40E_ERR_NO_MEMORY;
3016	}
3017
3018	/* Allocate TX hardware ring descriptors. */
3019	ring_size = sizeof(struct i40e_tx_desc) * I40E_FDIR_NUM_TX_DESC;
3020	ring_size = RTE_ALIGN(ring_size, I40E_DMA_MEM_ALIGN);
3021
3022	tz = rte_eth_dma_zone_reserve(dev, "fdir_tx_ring",
3023				      I40E_FDIR_QUEUE_ID, ring_size,
3024				      I40E_RING_BASE_ALIGN, SOCKET_ID_ANY);
3025	if (!tz) {
3026		i40e_dev_tx_queue_release(txq);
3027		PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for TX.");
3028		return I40E_ERR_NO_MEMORY;
3029	}
3030
3031	txq->nb_tx_desc = I40E_FDIR_NUM_TX_DESC;
3032	txq->queue_id = I40E_FDIR_QUEUE_ID;
3033	txq->reg_idx = pf->fdir.fdir_vsi->base_queue;
3034	txq->vsi = pf->fdir.fdir_vsi;
3035
3036	txq->tx_ring_phys_addr = rte_mem_phy2mch(tz->memseg_id, tz->phys_addr);
3037	txq->tx_ring = (struct i40e_tx_desc *)tz->addr;
3038	/*
3039	 * don't need to allocate software ring and reset for the fdir
3040	 * program queue just set the queue has been configured.
3041	 */
3042	txq->q_set = TRUE;
3043	pf->fdir.txq = txq;
3044
3045	return I40E_SUCCESS;
3046}
3047
3048enum i40e_status_code
3049i40e_fdir_setup_rx_resources(struct i40e_pf *pf)
3050{
3051	struct i40e_rx_queue *rxq;
3052	const struct rte_memzone *rz = NULL;
3053	uint32_t ring_size;
3054	struct rte_eth_dev *dev;
3055
3056	if (!pf) {
3057		PMD_DRV_LOG(ERR, "PF is not available");
3058		return I40E_ERR_BAD_PTR;
3059	}
3060
3061	dev = pf->adapter->eth_dev;
3062
3063	/* Allocate the RX queue data structure. */
3064	rxq = rte_zmalloc_socket("i40e fdir rx queue",
3065				  sizeof(struct i40e_rx_queue),
3066				  RTE_CACHE_LINE_SIZE,
3067				  SOCKET_ID_ANY);
3068	if (!rxq) {
3069		PMD_DRV_LOG(ERR, "Failed to allocate memory for "
3070					"rx queue structure.");
3071		return I40E_ERR_NO_MEMORY;
3072	}
3073
3074	/* Allocate RX hardware ring descriptors. */
3075	ring_size = sizeof(union i40e_rx_desc) * I40E_FDIR_NUM_RX_DESC;
3076	ring_size = RTE_ALIGN(ring_size, I40E_DMA_MEM_ALIGN);
3077
3078	rz = rte_eth_dma_zone_reserve(dev, "fdir_rx_ring",
3079				      I40E_FDIR_QUEUE_ID, ring_size,
3080				      I40E_RING_BASE_ALIGN, SOCKET_ID_ANY);
3081	if (!rz) {
3082		i40e_dev_rx_queue_release(rxq);
3083		PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for RX.");
3084		return I40E_ERR_NO_MEMORY;
3085	}
3086
3087	rxq->nb_rx_desc = I40E_FDIR_NUM_RX_DESC;
3088	rxq->queue_id = I40E_FDIR_QUEUE_ID;
3089	rxq->reg_idx = pf->fdir.fdir_vsi->base_queue;
3090	rxq->vsi = pf->fdir.fdir_vsi;
3091
3092	rxq->rx_ring_phys_addr = rte_mem_phy2mch(rz->memseg_id, rz->phys_addr);
3093	rxq->rx_ring = (union i40e_rx_desc *)rz->addr;
3094
3095	/*
3096	 * Don't need to allocate software ring and reset for the fdir
3097	 * rx queue, just set the queue has been configured.
3098	 */
3099	rxq->q_set = TRUE;
3100	pf->fdir.rxq = rxq;
3101
3102	return I40E_SUCCESS;
3103}
3104
3105void
3106i40e_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
3107	struct rte_eth_rxq_info *qinfo)
3108{
3109	struct i40e_rx_queue *rxq;
3110
3111	rxq = dev->data->rx_queues[queue_id];
3112
3113	qinfo->mp = rxq->mp;
3114	qinfo->scattered_rx = dev->data->scattered_rx;
3115	qinfo->nb_desc = rxq->nb_rx_desc;
3116
3117	qinfo->conf.rx_free_thresh = rxq->rx_free_thresh;
3118	qinfo->conf.rx_drop_en = rxq->drop_en;
3119	qinfo->conf.rx_deferred_start = rxq->rx_deferred_start;
3120}
3121
3122void
3123i40e_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
3124	struct rte_eth_txq_info *qinfo)
3125{
3126	struct i40e_tx_queue *txq;
3127
3128	txq = dev->data->tx_queues[queue_id];
3129
3130	qinfo->nb_desc = txq->nb_tx_desc;
3131
3132	qinfo->conf.tx_thresh.pthresh = txq->pthresh;
3133	qinfo->conf.tx_thresh.hthresh = txq->hthresh;
3134	qinfo->conf.tx_thresh.wthresh = txq->wthresh;
3135
3136	qinfo->conf.tx_free_thresh = txq->tx_free_thresh;
3137	qinfo->conf.tx_rs_thresh = txq->tx_rs_thresh;
3138	qinfo->conf.txq_flags = txq->txq_flags;
3139	qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
3140}
3141
3142void __attribute__((cold))
3143i40e_set_rx_function(struct rte_eth_dev *dev)
3144{
3145	struct i40e_adapter *ad =
3146		I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
3147	uint16_t rx_using_sse, i;
3148	/* In order to allow Vector Rx there are a few configuration
3149	 * conditions to be met and Rx Bulk Allocation should be allowed.
3150	 */
3151	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
3152		if (i40e_rx_vec_dev_conf_condition_check(dev) ||
3153		    !ad->rx_bulk_alloc_allowed) {
3154			PMD_INIT_LOG(DEBUG, "Port[%d] doesn't meet"
3155				     " Vector Rx preconditions",
3156				     dev->data->port_id);
3157
3158			ad->rx_vec_allowed = false;
3159		}
3160		if (ad->rx_vec_allowed) {
3161			for (i = 0; i < dev->data->nb_rx_queues; i++) {
3162				struct i40e_rx_queue *rxq =
3163					dev->data->rx_queues[i];
3164
3165				if (rxq && i40e_rxq_vec_setup(rxq)) {
3166					ad->rx_vec_allowed = false;
3167					break;
3168				}
3169			}
3170		}
3171	}
3172
3173	if (dev->data->scattered_rx) {
3174		/* Set the non-LRO scattered callback: there are Vector and
3175		 * single allocation versions.
3176		 */
3177		if (ad->rx_vec_allowed) {
3178			PMD_INIT_LOG(DEBUG, "Using Vector Scattered Rx "
3179					    "callback (port=%d).",
3180				     dev->data->port_id);
3181
3182			dev->rx_pkt_burst = i40e_recv_scattered_pkts_vec;
3183		} else {
3184			PMD_INIT_LOG(DEBUG, "Using a Scattered with bulk "
3185					   "allocation callback (port=%d).",
3186				     dev->data->port_id);
3187			dev->rx_pkt_burst = i40e_recv_scattered_pkts;
3188		}
3189	/* If parameters allow we are going to choose between the following
3190	 * callbacks:
3191	 *    - Vector
3192	 *    - Bulk Allocation
3193	 *    - Single buffer allocation (the simplest one)
3194	 */
3195	} else if (ad->rx_vec_allowed) {
3196		PMD_INIT_LOG(DEBUG, "Vector rx enabled, please make sure RX "
3197				    "burst size no less than %d (port=%d).",
3198			     RTE_I40E_DESCS_PER_LOOP,
3199			     dev->data->port_id);
3200
3201		dev->rx_pkt_burst = i40e_recv_pkts_vec;
3202	} else if (ad->rx_bulk_alloc_allowed) {
3203		PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
3204				    "satisfied. Rx Burst Bulk Alloc function "
3205				    "will be used on port=%d.",
3206			     dev->data->port_id);
3207
3208		dev->rx_pkt_burst = i40e_recv_pkts_bulk_alloc;
3209	} else {
3210		PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are not "
3211				    "satisfied, or Scattered Rx is requested "
3212				    "(port=%d).",
3213			     dev->data->port_id);
3214
3215		dev->rx_pkt_burst = i40e_recv_pkts;
3216	}
3217
3218	/* Propagate information about RX function choice through all queues. */
3219	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
3220		rx_using_sse =
3221			(dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec ||
3222			 dev->rx_pkt_burst == i40e_recv_pkts_vec);
3223
3224		for (i = 0; i < dev->data->nb_rx_queues; i++) {
3225			struct i40e_rx_queue *rxq = dev->data->rx_queues[i];
3226
3227			if (rxq)
3228				rxq->rx_using_sse = rx_using_sse;
3229		}
3230	}
3231}
3232
3233void __attribute__((cold))
3234i40e_set_tx_function_flag(struct rte_eth_dev *dev, struct i40e_tx_queue *txq)
3235{
3236	struct i40e_adapter *ad =
3237		I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
3238
3239	/* Use a simple Tx queue (no offloads, no multi segs) if possible */
3240	if (((txq->txq_flags & I40E_SIMPLE_FLAGS) == I40E_SIMPLE_FLAGS)
3241			&& (txq->tx_rs_thresh >= RTE_PMD_I40E_TX_MAX_BURST)) {
3242		if (txq->tx_rs_thresh <= RTE_I40E_TX_MAX_FREE_BUF_SZ) {
3243			PMD_INIT_LOG(DEBUG, "Vector tx"
3244				     " can be enabled on this txq.");
3245
3246		} else {
3247			ad->tx_vec_allowed = false;
3248		}
3249	} else {
3250		ad->tx_simple_allowed = false;
3251	}
3252}
3253
3254void __attribute__((cold))
3255i40e_set_tx_function(struct rte_eth_dev *dev)
3256{
3257	struct i40e_adapter *ad =
3258		I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
3259	int i;
3260
3261	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
3262		if (ad->tx_vec_allowed) {
3263			for (i = 0; i < dev->data->nb_tx_queues; i++) {
3264				struct i40e_tx_queue *txq =
3265					dev->data->tx_queues[i];
3266
3267				if (txq && i40e_txq_vec_setup(txq)) {
3268					ad->tx_vec_allowed = false;
3269					break;
3270				}
3271			}
3272		}
3273	}
3274
3275	if (ad->tx_simple_allowed) {
3276		if (ad->tx_vec_allowed) {
3277			PMD_INIT_LOG(DEBUG, "Vector tx finally be used.");
3278			dev->tx_pkt_burst = i40e_xmit_pkts_vec;
3279		} else {
3280			PMD_INIT_LOG(DEBUG, "Simple tx finally be used.");
3281			dev->tx_pkt_burst = i40e_xmit_pkts_simple;
3282		}
3283	} else {
3284		PMD_INIT_LOG(DEBUG, "Xmit tx finally be used.");
3285		dev->tx_pkt_burst = i40e_xmit_pkts;
3286	}
3287}
3288
3289/* Stubs needed for linkage when CONFIG_RTE_I40E_INC_VECTOR is set to 'n' */
3290int __attribute__((weak))
3291i40e_rx_vec_dev_conf_condition_check(struct rte_eth_dev __rte_unused *dev)
3292{
3293	return -1;
3294}
3295
3296uint16_t __attribute__((weak))
3297i40e_recv_pkts_vec(
3298	void __rte_unused *rx_queue,
3299	struct rte_mbuf __rte_unused **rx_pkts,
3300	uint16_t __rte_unused nb_pkts)
3301{
3302	return 0;
3303}
3304
3305uint16_t __attribute__((weak))
3306i40e_recv_scattered_pkts_vec(
3307	void __rte_unused *rx_queue,
3308	struct rte_mbuf __rte_unused **rx_pkts,
3309	uint16_t __rte_unused nb_pkts)
3310{
3311	return 0;
3312}
3313
3314int __attribute__((weak))
3315i40e_rxq_vec_setup(struct i40e_rx_queue __rte_unused *rxq)
3316{
3317	return -1;
3318}
3319
3320int __attribute__((weak))
3321i40e_txq_vec_setup(struct i40e_tx_queue __rte_unused *txq)
3322{
3323	return -1;
3324}
3325
3326void __attribute__((weak))
3327i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue __rte_unused*rxq)
3328{
3329	return;
3330}
3331
3332uint16_t __attribute__((weak))
3333i40e_xmit_pkts_vec(void __rte_unused *tx_queue,
3334		   struct rte_mbuf __rte_unused **tx_pkts,
3335		   uint16_t __rte_unused nb_pkts)
3336{
3337	return 0;
3338}
3339