i40e_rxtx.c revision 93f15e30
1/*-
2 *   BSD LICENSE
3 *
4 *   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
5 *   All rights reserved.
6 *
7 *   Redistribution and use in source and binary forms, with or without
8 *   modification, are permitted provided that the following conditions
9 *   are met:
10 *
11 *     * Redistributions of source code must retain the above copyright
12 *       notice, this list of conditions and the following disclaimer.
13 *     * Redistributions in binary form must reproduce the above copyright
14 *       notice, this list of conditions and the following disclaimer in
15 *       the documentation and/or other materials provided with the
16 *       distribution.
17 *     * Neither the name of Intel Corporation nor the names of its
18 *       contributors may be used to endorse or promote products derived
19 *       from this software without specific prior written permission.
20 *
21 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34#include <stdio.h>
35#include <stdlib.h>
36#include <string.h>
37#include <errno.h>
38#include <stdint.h>
39#include <stdarg.h>
40#include <unistd.h>
41#include <inttypes.h>
42#include <sys/queue.h>
43
44#include <rte_string_fns.h>
45#include <rte_memzone.h>
46#include <rte_mbuf.h>
47#include <rte_malloc.h>
48#include <rte_ether.h>
49#include <rte_ethdev.h>
50#include <rte_tcp.h>
51#include <rte_sctp.h>
52#include <rte_udp.h>
53#include <rte_ip.h>
54#include <rte_net.h>
55
56#include "i40e_logs.h"
57#include "base/i40e_prototype.h"
58#include "base/i40e_type.h"
59#include "i40e_ethdev.h"
60#include "i40e_rxtx.h"
61
62#define DEFAULT_TX_RS_THRESH   32
63#define DEFAULT_TX_FREE_THRESH 32
64#define I40E_MAX_PKT_TYPE      256
65
66#define I40E_TX_MAX_BURST  32
67
68#define I40E_DMA_MEM_ALIGN 4096
69
70/* Base address of the HW descriptor ring should be 128B aligned. */
71#define I40E_RING_BASE_ALIGN	128
72
73#define I40E_SIMPLE_FLAGS ((uint32_t)ETH_TXQ_FLAGS_NOMULTSEGS | \
74					ETH_TXQ_FLAGS_NOOFFLOADS)
75
76#define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS)
77
78#ifdef RTE_LIBRTE_IEEE1588
79#define I40E_TX_IEEE1588_TMST PKT_TX_IEEE1588_TMST
80#else
81#define I40E_TX_IEEE1588_TMST 0
82#endif
83
84#define I40E_TX_CKSUM_OFFLOAD_MASK (		 \
85		PKT_TX_IP_CKSUM |		 \
86		PKT_TX_L4_MASK |		 \
87		PKT_TX_TCP_SEG |		 \
88		PKT_TX_OUTER_IP_CKSUM)
89
90#define I40E_TX_OFFLOAD_MASK (  \
91		PKT_TX_IP_CKSUM |       \
92		PKT_TX_L4_MASK |        \
93		PKT_TX_OUTER_IP_CKSUM | \
94		PKT_TX_TCP_SEG |        \
95		PKT_TX_QINQ_PKT |       \
96		PKT_TX_VLAN_PKT |	\
97		PKT_TX_TUNNEL_MASK |	\
98		I40E_TX_IEEE1588_TMST)
99
100#define I40E_TX_OFFLOAD_NOTSUP_MASK \
101		(PKT_TX_OFFLOAD_MASK ^ I40E_TX_OFFLOAD_MASK)
102
103static uint16_t i40e_xmit_pkts_simple(void *tx_queue,
104				      struct rte_mbuf **tx_pkts,
105				      uint16_t nb_pkts);
106
107static inline void
108i40e_rxd_to_vlan_tci(struct rte_mbuf *mb, volatile union i40e_rx_desc *rxdp)
109{
110	if (rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len) &
111		(1 << I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)) {
112		mb->ol_flags |= PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED;
113		mb->vlan_tci =
114			rte_le_to_cpu_16(rxdp->wb.qword0.lo_dword.l2tag1);
115		PMD_RX_LOG(DEBUG, "Descriptor l2tag1: %u",
116			   rte_le_to_cpu_16(rxdp->wb.qword0.lo_dword.l2tag1));
117	} else {
118		mb->vlan_tci = 0;
119	}
120#ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC
121	if (rte_le_to_cpu_16(rxdp->wb.qword2.ext_status) &
122		(1 << I40E_RX_DESC_EXT_STATUS_L2TAG2P_SHIFT)) {
123		mb->ol_flags |= PKT_RX_QINQ_STRIPPED;
124		mb->vlan_tci_outer = mb->vlan_tci;
125		mb->vlan_tci = rte_le_to_cpu_16(rxdp->wb.qword2.l2tag2_2);
126		PMD_RX_LOG(DEBUG, "Descriptor l2tag2_1: %u, l2tag2_2: %u",
127			   rte_le_to_cpu_16(rxdp->wb.qword2.l2tag2_1),
128			   rte_le_to_cpu_16(rxdp->wb.qword2.l2tag2_2));
129	} else {
130		mb->vlan_tci_outer = 0;
131	}
132#endif
133	PMD_RX_LOG(DEBUG, "Mbuf vlan_tci: %u, vlan_tci_outer: %u",
134		   mb->vlan_tci, mb->vlan_tci_outer);
135}
136
137/* Translate the rx descriptor status to pkt flags */
138static inline uint64_t
139i40e_rxd_status_to_pkt_flags(uint64_t qword)
140{
141	uint64_t flags;
142
143	/* Check if RSS_HASH */
144	flags = (((qword >> I40E_RX_DESC_STATUS_FLTSTAT_SHIFT) &
145					I40E_RX_DESC_FLTSTAT_RSS_HASH) ==
146			I40E_RX_DESC_FLTSTAT_RSS_HASH) ? PKT_RX_RSS_HASH : 0;
147
148	/* Check if FDIR Match */
149	flags |= (qword & (1 << I40E_RX_DESC_STATUS_FLM_SHIFT) ?
150							PKT_RX_FDIR : 0);
151
152	return flags;
153}
154
155static inline uint64_t
156i40e_rxd_error_to_pkt_flags(uint64_t qword)
157{
158	uint64_t flags = 0;
159	uint64_t error_bits = (qword >> I40E_RXD_QW1_ERROR_SHIFT);
160
161#define I40E_RX_ERR_BITS 0x3f
162	if (likely((error_bits & I40E_RX_ERR_BITS) == 0)) {
163		flags |= (PKT_RX_IP_CKSUM_GOOD | PKT_RX_L4_CKSUM_GOOD);
164		return flags;
165	}
166
167	if (unlikely(error_bits & (1 << I40E_RX_DESC_ERROR_IPE_SHIFT)))
168		flags |= PKT_RX_IP_CKSUM_BAD;
169	else
170		flags |= PKT_RX_IP_CKSUM_GOOD;
171
172	if (unlikely(error_bits & (1 << I40E_RX_DESC_ERROR_L4E_SHIFT)))
173		flags |= PKT_RX_L4_CKSUM_BAD;
174	else
175		flags |= PKT_RX_L4_CKSUM_GOOD;
176
177	if (unlikely(error_bits & (1 << I40E_RX_DESC_ERROR_EIPE_SHIFT)))
178		flags |= PKT_RX_EIP_CKSUM_BAD;
179
180	return flags;
181}
182
183/* Function to check and set the ieee1588 timesync index and get the
184 * appropriate flags.
185 */
186#ifdef RTE_LIBRTE_IEEE1588
187static inline uint64_t
188i40e_get_iee15888_flags(struct rte_mbuf *mb, uint64_t qword)
189{
190	uint64_t pkt_flags = 0;
191	uint16_t tsyn = (qword & (I40E_RXD_QW1_STATUS_TSYNVALID_MASK
192				  | I40E_RXD_QW1_STATUS_TSYNINDX_MASK))
193				    >> I40E_RX_DESC_STATUS_TSYNINDX_SHIFT;
194
195	if ((mb->packet_type & RTE_PTYPE_L2_MASK)
196			== RTE_PTYPE_L2_ETHER_TIMESYNC)
197		pkt_flags = PKT_RX_IEEE1588_PTP;
198	if (tsyn & 0x04) {
199		pkt_flags |= PKT_RX_IEEE1588_TMST;
200		mb->timesync = tsyn & 0x03;
201	}
202
203	return pkt_flags;
204}
205#endif
206
207#define I40E_RX_DESC_EXT_STATUS_FLEXBH_MASK   0x03
208#define I40E_RX_DESC_EXT_STATUS_FLEXBH_FD_ID  0x01
209#define I40E_RX_DESC_EXT_STATUS_FLEXBH_FLEX   0x02
210#define I40E_RX_DESC_EXT_STATUS_FLEXBL_MASK   0x03
211#define I40E_RX_DESC_EXT_STATUS_FLEXBL_FLEX   0x01
212
213static inline uint64_t
214i40e_rxd_build_fdir(volatile union i40e_rx_desc *rxdp, struct rte_mbuf *mb)
215{
216	uint64_t flags = 0;
217#ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC
218	uint16_t flexbh, flexbl;
219
220	flexbh = (rte_le_to_cpu_32(rxdp->wb.qword2.ext_status) >>
221		I40E_RX_DESC_EXT_STATUS_FLEXBH_SHIFT) &
222		I40E_RX_DESC_EXT_STATUS_FLEXBH_MASK;
223	flexbl = (rte_le_to_cpu_32(rxdp->wb.qword2.ext_status) >>
224		I40E_RX_DESC_EXT_STATUS_FLEXBL_SHIFT) &
225		I40E_RX_DESC_EXT_STATUS_FLEXBL_MASK;
226
227
228	if (flexbh == I40E_RX_DESC_EXT_STATUS_FLEXBH_FD_ID) {
229		mb->hash.fdir.hi =
230			rte_le_to_cpu_32(rxdp->wb.qword3.hi_dword.fd_id);
231		flags |= PKT_RX_FDIR_ID;
232	} else if (flexbh == I40E_RX_DESC_EXT_STATUS_FLEXBH_FLEX) {
233		mb->hash.fdir.hi =
234			rte_le_to_cpu_32(rxdp->wb.qword3.hi_dword.flex_bytes_hi);
235		flags |= PKT_RX_FDIR_FLX;
236	}
237	if (flexbl == I40E_RX_DESC_EXT_STATUS_FLEXBL_FLEX) {
238		mb->hash.fdir.lo =
239			rte_le_to_cpu_32(rxdp->wb.qword3.lo_dword.flex_bytes_lo);
240		flags |= PKT_RX_FDIR_FLX;
241	}
242#else
243	mb->hash.fdir.hi =
244		rte_le_to_cpu_32(rxdp->wb.qword0.hi_dword.fd_id);
245	flags |= PKT_RX_FDIR_ID;
246#endif
247	return flags;
248}
249
250static inline void
251i40e_parse_tunneling_params(uint64_t ol_flags,
252			    union i40e_tx_offload tx_offload,
253			    uint32_t *cd_tunneling)
254{
255	/* EIPT: External (outer) IP header type */
256	if (ol_flags & PKT_TX_OUTER_IP_CKSUM)
257		*cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV4;
258	else if (ol_flags & PKT_TX_OUTER_IPV4)
259		*cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM;
260	else if (ol_flags & PKT_TX_OUTER_IPV6)
261		*cd_tunneling |= I40E_TX_CTX_EXT_IP_IPV6;
262
263	/* EIPLEN: External (outer) IP header length, in DWords */
264	*cd_tunneling |= (tx_offload.outer_l3_len >> 2) <<
265		I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT;
266
267	/* L4TUNT: L4 Tunneling Type */
268	switch (ol_flags & PKT_TX_TUNNEL_MASK) {
269	case PKT_TX_TUNNEL_IPIP:
270		/* for non UDP / GRE tunneling, set to 00b */
271		break;
272	case PKT_TX_TUNNEL_VXLAN:
273	case PKT_TX_TUNNEL_GENEVE:
274		*cd_tunneling |= I40E_TXD_CTX_UDP_TUNNELING;
275		break;
276	case PKT_TX_TUNNEL_GRE:
277		*cd_tunneling |= I40E_TXD_CTX_GRE_TUNNELING;
278		break;
279	default:
280		PMD_TX_LOG(ERR, "Tunnel type not supported");
281		return;
282	}
283
284	/* L4TUNLEN: L4 Tunneling Length, in Words
285	 *
286	 * We depend on app to set rte_mbuf.l2_len correctly.
287	 * For IP in GRE it should be set to the length of the GRE
288	 * header;
289	 * for MAC in GRE or MAC in UDP it should be set to the length
290	 * of the GRE or UDP headers plus the inner MAC up to including
291	 * its last Ethertype.
292	 */
293	*cd_tunneling |= (tx_offload.l2_len >> 1) <<
294		I40E_TXD_CTX_QW0_NATLEN_SHIFT;
295}
296
297static inline void
298i40e_txd_enable_checksum(uint64_t ol_flags,
299			uint32_t *td_cmd,
300			uint32_t *td_offset,
301			union i40e_tx_offload tx_offload)
302{
303	/* Set MACLEN */
304	if (ol_flags & PKT_TX_TUNNEL_MASK)
305		*td_offset |= (tx_offload.outer_l2_len >> 1)
306				<< I40E_TX_DESC_LENGTH_MACLEN_SHIFT;
307	else
308		*td_offset |= (tx_offload.l2_len >> 1)
309			<< I40E_TX_DESC_LENGTH_MACLEN_SHIFT;
310
311	/* Enable L3 checksum offloads */
312	if (ol_flags & PKT_TX_IP_CKSUM) {
313		*td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4_CSUM;
314		*td_offset |= (tx_offload.l3_len >> 2)
315				<< I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
316	} else if (ol_flags & PKT_TX_IPV4) {
317		*td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV4;
318		*td_offset |= (tx_offload.l3_len >> 2)
319				<< I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
320	} else if (ol_flags & PKT_TX_IPV6) {
321		*td_cmd |= I40E_TX_DESC_CMD_IIPT_IPV6;
322		*td_offset |= (tx_offload.l3_len >> 2)
323				<< I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
324	}
325
326	if (ol_flags & PKT_TX_TCP_SEG) {
327		*td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP;
328		*td_offset |= (tx_offload.l4_len >> 2)
329			<< I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
330		return;
331	}
332
333	/* Enable L4 checksum offloads */
334	switch (ol_flags & PKT_TX_L4_MASK) {
335	case PKT_TX_TCP_CKSUM:
336		*td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP;
337		*td_offset |= (sizeof(struct tcp_hdr) >> 2) <<
338				I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
339		break;
340	case PKT_TX_SCTP_CKSUM:
341		*td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP;
342		*td_offset |= (sizeof(struct sctp_hdr) >> 2) <<
343				I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
344		break;
345	case PKT_TX_UDP_CKSUM:
346		*td_cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP;
347		*td_offset |= (sizeof(struct udp_hdr) >> 2) <<
348				I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
349		break;
350	default:
351		break;
352	}
353}
354
355/* Construct the tx flags */
356static inline uint64_t
357i40e_build_ctob(uint32_t td_cmd,
358		uint32_t td_offset,
359		unsigned int size,
360		uint32_t td_tag)
361{
362	return rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DATA |
363			((uint64_t)td_cmd  << I40E_TXD_QW1_CMD_SHIFT) |
364			((uint64_t)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) |
365			((uint64_t)size  << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) |
366			((uint64_t)td_tag  << I40E_TXD_QW1_L2TAG1_SHIFT));
367}
368
369static inline int
370i40e_xmit_cleanup(struct i40e_tx_queue *txq)
371{
372	struct i40e_tx_entry *sw_ring = txq->sw_ring;
373	volatile struct i40e_tx_desc *txd = txq->tx_ring;
374	uint16_t last_desc_cleaned = txq->last_desc_cleaned;
375	uint16_t nb_tx_desc = txq->nb_tx_desc;
376	uint16_t desc_to_clean_to;
377	uint16_t nb_tx_to_clean;
378
379	desc_to_clean_to = (uint16_t)(last_desc_cleaned + txq->tx_rs_thresh);
380	if (desc_to_clean_to >= nb_tx_desc)
381		desc_to_clean_to = (uint16_t)(desc_to_clean_to - nb_tx_desc);
382
383	desc_to_clean_to = sw_ring[desc_to_clean_to].last_id;
384	if ((txd[desc_to_clean_to].cmd_type_offset_bsz &
385			rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) !=
386			rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE)) {
387		PMD_TX_FREE_LOG(DEBUG, "TX descriptor %4u is not done "
388			"(port=%d queue=%d)", desc_to_clean_to,
389				txq->port_id, txq->queue_id);
390		return -1;
391	}
392
393	if (last_desc_cleaned > desc_to_clean_to)
394		nb_tx_to_clean = (uint16_t)((nb_tx_desc - last_desc_cleaned) +
395							desc_to_clean_to);
396	else
397		nb_tx_to_clean = (uint16_t)(desc_to_clean_to -
398					last_desc_cleaned);
399
400	txd[desc_to_clean_to].cmd_type_offset_bsz = 0;
401
402	txq->last_desc_cleaned = desc_to_clean_to;
403	txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + nb_tx_to_clean);
404
405	return 0;
406}
407
408static inline int
409#ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC
410check_rx_burst_bulk_alloc_preconditions(struct i40e_rx_queue *rxq)
411#else
412check_rx_burst_bulk_alloc_preconditions(__rte_unused struct i40e_rx_queue *rxq)
413#endif
414{
415	int ret = 0;
416
417#ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC
418	if (!(rxq->rx_free_thresh >= RTE_PMD_I40E_RX_MAX_BURST)) {
419		PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
420			     "rxq->rx_free_thresh=%d, "
421			     "RTE_PMD_I40E_RX_MAX_BURST=%d",
422			     rxq->rx_free_thresh, RTE_PMD_I40E_RX_MAX_BURST);
423		ret = -EINVAL;
424	} else if (!(rxq->rx_free_thresh < rxq->nb_rx_desc)) {
425		PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
426			     "rxq->rx_free_thresh=%d, "
427			     "rxq->nb_rx_desc=%d",
428			     rxq->rx_free_thresh, rxq->nb_rx_desc);
429		ret = -EINVAL;
430	} else if (rxq->nb_rx_desc % rxq->rx_free_thresh != 0) {
431		PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions: "
432			     "rxq->nb_rx_desc=%d, "
433			     "rxq->rx_free_thresh=%d",
434			     rxq->nb_rx_desc, rxq->rx_free_thresh);
435		ret = -EINVAL;
436	}
437#else
438	ret = -EINVAL;
439#endif
440
441	return ret;
442}
443
444#ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC
445#define I40E_LOOK_AHEAD 8
446#if (I40E_LOOK_AHEAD != 8)
447#error "PMD I40E: I40E_LOOK_AHEAD must be 8\n"
448#endif
449static inline int
450i40e_rx_scan_hw_ring(struct i40e_rx_queue *rxq)
451{
452	volatile union i40e_rx_desc *rxdp;
453	struct i40e_rx_entry *rxep;
454	struct rte_mbuf *mb;
455	uint16_t pkt_len;
456	uint64_t qword1;
457	uint32_t rx_status;
458	int32_t s[I40E_LOOK_AHEAD], nb_dd;
459	int32_t i, j, nb_rx = 0;
460	uint64_t pkt_flags;
461
462	rxdp = &rxq->rx_ring[rxq->rx_tail];
463	rxep = &rxq->sw_ring[rxq->rx_tail];
464
465	qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len);
466	rx_status = (qword1 & I40E_RXD_QW1_STATUS_MASK) >>
467				I40E_RXD_QW1_STATUS_SHIFT;
468
469	/* Make sure there is at least 1 packet to receive */
470	if (!(rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)))
471		return 0;
472
473	/**
474	 * Scan LOOK_AHEAD descriptors at a time to determine which
475	 * descriptors reference packets that are ready to be received.
476	 */
477	for (i = 0; i < RTE_PMD_I40E_RX_MAX_BURST; i+=I40E_LOOK_AHEAD,
478			rxdp += I40E_LOOK_AHEAD, rxep += I40E_LOOK_AHEAD) {
479		/* Read desc statuses backwards to avoid race condition */
480		for (j = I40E_LOOK_AHEAD - 1; j >= 0; j--) {
481			qword1 = rte_le_to_cpu_64(\
482				rxdp[j].wb.qword1.status_error_len);
483			s[j] = (qword1 & I40E_RXD_QW1_STATUS_MASK) >>
484					I40E_RXD_QW1_STATUS_SHIFT;
485		}
486
487		rte_smp_rmb();
488
489		/* Compute how many status bits were set */
490		for (j = 0, nb_dd = 0; j < I40E_LOOK_AHEAD; j++)
491			nb_dd += s[j] & (1 << I40E_RX_DESC_STATUS_DD_SHIFT);
492
493		nb_rx += nb_dd;
494
495		/* Translate descriptor info to mbuf parameters */
496		for (j = 0; j < nb_dd; j++) {
497			mb = rxep[j].mbuf;
498			qword1 = rte_le_to_cpu_64(\
499				rxdp[j].wb.qword1.status_error_len);
500			pkt_len = ((qword1 & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
501				I40E_RXD_QW1_LENGTH_PBUF_SHIFT) - rxq->crc_len;
502			mb->data_len = pkt_len;
503			mb->pkt_len = pkt_len;
504			mb->ol_flags = 0;
505			i40e_rxd_to_vlan_tci(mb, &rxdp[j]);
506			pkt_flags = i40e_rxd_status_to_pkt_flags(qword1);
507			pkt_flags |= i40e_rxd_error_to_pkt_flags(qword1);
508			mb->packet_type =
509				i40e_rxd_pkt_type_mapping((uint8_t)((qword1 &
510						I40E_RXD_QW1_PTYPE_MASK) >>
511						I40E_RXD_QW1_PTYPE_SHIFT));
512			if (pkt_flags & PKT_RX_RSS_HASH)
513				mb->hash.rss = rte_le_to_cpu_32(\
514					rxdp[j].wb.qword0.hi_dword.rss);
515			if (pkt_flags & PKT_RX_FDIR)
516				pkt_flags |= i40e_rxd_build_fdir(&rxdp[j], mb);
517
518#ifdef RTE_LIBRTE_IEEE1588
519			pkt_flags |= i40e_get_iee15888_flags(mb, qword1);
520#endif
521			mb->ol_flags |= pkt_flags;
522
523		}
524
525		for (j = 0; j < I40E_LOOK_AHEAD; j++)
526			rxq->rx_stage[i + j] = rxep[j].mbuf;
527
528		if (nb_dd != I40E_LOOK_AHEAD)
529			break;
530	}
531
532	/* Clear software ring entries */
533	for (i = 0; i < nb_rx; i++)
534		rxq->sw_ring[rxq->rx_tail + i].mbuf = NULL;
535
536	return nb_rx;
537}
538
539static inline uint16_t
540i40e_rx_fill_from_stage(struct i40e_rx_queue *rxq,
541			struct rte_mbuf **rx_pkts,
542			uint16_t nb_pkts)
543{
544	uint16_t i;
545	struct rte_mbuf **stage = &rxq->rx_stage[rxq->rx_next_avail];
546
547	nb_pkts = (uint16_t)RTE_MIN(nb_pkts, rxq->rx_nb_avail);
548
549	for (i = 0; i < nb_pkts; i++)
550		rx_pkts[i] = stage[i];
551
552	rxq->rx_nb_avail = (uint16_t)(rxq->rx_nb_avail - nb_pkts);
553	rxq->rx_next_avail = (uint16_t)(rxq->rx_next_avail + nb_pkts);
554
555	return nb_pkts;
556}
557
558static inline int
559i40e_rx_alloc_bufs(struct i40e_rx_queue *rxq)
560{
561	volatile union i40e_rx_desc *rxdp;
562	struct i40e_rx_entry *rxep;
563	struct rte_mbuf *mb;
564	uint16_t alloc_idx, i;
565	uint64_t dma_addr;
566	int diag;
567
568	/* Allocate buffers in bulk */
569	alloc_idx = (uint16_t)(rxq->rx_free_trigger -
570				(rxq->rx_free_thresh - 1));
571	rxep = &(rxq->sw_ring[alloc_idx]);
572	diag = rte_mempool_get_bulk(rxq->mp, (void *)rxep,
573					rxq->rx_free_thresh);
574	if (unlikely(diag != 0)) {
575		PMD_DRV_LOG(ERR, "Failed to get mbufs in bulk");
576		return -ENOMEM;
577	}
578
579	rxdp = &rxq->rx_ring[alloc_idx];
580	for (i = 0; i < rxq->rx_free_thresh; i++) {
581		if (likely(i < (rxq->rx_free_thresh - 1)))
582			/* Prefetch next mbuf */
583			rte_prefetch0(rxep[i + 1].mbuf);
584
585		mb = rxep[i].mbuf;
586		rte_mbuf_refcnt_set(mb, 1);
587		mb->next = NULL;
588		mb->data_off = RTE_PKTMBUF_HEADROOM;
589		mb->nb_segs = 1;
590		mb->port = rxq->port_id;
591		dma_addr = rte_cpu_to_le_64(\
592			rte_mbuf_data_dma_addr_default(mb));
593		rxdp[i].read.hdr_addr = 0;
594		rxdp[i].read.pkt_addr = dma_addr;
595	}
596
597	/* Update rx tail regsiter */
598	rte_wmb();
599	I40E_PCI_REG_WRITE_RELAXED(rxq->qrx_tail, rxq->rx_free_trigger);
600
601	rxq->rx_free_trigger =
602		(uint16_t)(rxq->rx_free_trigger + rxq->rx_free_thresh);
603	if (rxq->rx_free_trigger >= rxq->nb_rx_desc)
604		rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1);
605
606	return 0;
607}
608
609static inline uint16_t
610rx_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
611{
612	struct i40e_rx_queue *rxq = (struct i40e_rx_queue *)rx_queue;
613	uint16_t nb_rx = 0;
614
615	if (!nb_pkts)
616		return 0;
617
618	if (rxq->rx_nb_avail)
619		return i40e_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
620
621	nb_rx = (uint16_t)i40e_rx_scan_hw_ring(rxq);
622	rxq->rx_next_avail = 0;
623	rxq->rx_nb_avail = nb_rx;
624	rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_rx);
625
626	if (rxq->rx_tail > rxq->rx_free_trigger) {
627		if (i40e_rx_alloc_bufs(rxq) != 0) {
628			uint16_t i, j;
629
630			PMD_RX_LOG(DEBUG, "Rx mbuf alloc failed for "
631				   "port_id=%u, queue_id=%u",
632				   rxq->port_id, rxq->queue_id);
633			rxq->rx_nb_avail = 0;
634			rxq->rx_tail = (uint16_t)(rxq->rx_tail - nb_rx);
635			for (i = 0, j = rxq->rx_tail; i < nb_rx; i++, j++)
636				rxq->sw_ring[j].mbuf = rxq->rx_stage[i];
637
638			return 0;
639		}
640	}
641
642	if (rxq->rx_tail >= rxq->nb_rx_desc)
643		rxq->rx_tail = 0;
644
645	if (rxq->rx_nb_avail)
646		return i40e_rx_fill_from_stage(rxq, rx_pkts, nb_pkts);
647
648	return 0;
649}
650
651static uint16_t
652i40e_recv_pkts_bulk_alloc(void *rx_queue,
653			  struct rte_mbuf **rx_pkts,
654			  uint16_t nb_pkts)
655{
656	uint16_t nb_rx = 0, n, count;
657
658	if (unlikely(nb_pkts == 0))
659		return 0;
660
661	if (likely(nb_pkts <= RTE_PMD_I40E_RX_MAX_BURST))
662		return rx_recv_pkts(rx_queue, rx_pkts, nb_pkts);
663
664	while (nb_pkts) {
665		n = RTE_MIN(nb_pkts, RTE_PMD_I40E_RX_MAX_BURST);
666		count = rx_recv_pkts(rx_queue, &rx_pkts[nb_rx], n);
667		nb_rx = (uint16_t)(nb_rx + count);
668		nb_pkts = (uint16_t)(nb_pkts - count);
669		if (count < n)
670			break;
671	}
672
673	return nb_rx;
674}
675#else
676static uint16_t
677i40e_recv_pkts_bulk_alloc(void __rte_unused *rx_queue,
678			  struct rte_mbuf __rte_unused **rx_pkts,
679			  uint16_t __rte_unused nb_pkts)
680{
681	return 0;
682}
683#endif /* RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC */
684
685uint16_t
686i40e_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
687{
688	struct i40e_rx_queue *rxq;
689	volatile union i40e_rx_desc *rx_ring;
690	volatile union i40e_rx_desc *rxdp;
691	union i40e_rx_desc rxd;
692	struct i40e_rx_entry *sw_ring;
693	struct i40e_rx_entry *rxe;
694	struct rte_mbuf *rxm;
695	struct rte_mbuf *nmb;
696	uint16_t nb_rx;
697	uint32_t rx_status;
698	uint64_t qword1;
699	uint16_t rx_packet_len;
700	uint16_t rx_id, nb_hold;
701	uint64_t dma_addr;
702	uint64_t pkt_flags;
703
704	nb_rx = 0;
705	nb_hold = 0;
706	rxq = rx_queue;
707	rx_id = rxq->rx_tail;
708	rx_ring = rxq->rx_ring;
709	sw_ring = rxq->sw_ring;
710
711	while (nb_rx < nb_pkts) {
712		rxdp = &rx_ring[rx_id];
713		qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len);
714		rx_status = (qword1 & I40E_RXD_QW1_STATUS_MASK)
715				>> I40E_RXD_QW1_STATUS_SHIFT;
716
717		/* Check the DD bit first */
718		if (!(rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)))
719			break;
720
721		nmb = rte_mbuf_raw_alloc(rxq->mp);
722		if (unlikely(!nmb))
723			break;
724		rxd = *rxdp;
725
726		nb_hold++;
727		rxe = &sw_ring[rx_id];
728		rx_id++;
729		if (unlikely(rx_id == rxq->nb_rx_desc))
730			rx_id = 0;
731
732		/* Prefetch next mbuf */
733		rte_prefetch0(sw_ring[rx_id].mbuf);
734
735		/**
736		 * When next RX descriptor is on a cache line boundary,
737		 * prefetch the next 4 RX descriptors and next 8 pointers
738		 * to mbufs.
739		 */
740		if ((rx_id & 0x3) == 0) {
741			rte_prefetch0(&rx_ring[rx_id]);
742			rte_prefetch0(&sw_ring[rx_id]);
743		}
744		rxm = rxe->mbuf;
745		rxe->mbuf = nmb;
746		dma_addr =
747			rte_cpu_to_le_64(rte_mbuf_data_dma_addr_default(nmb));
748		rxdp->read.hdr_addr = 0;
749		rxdp->read.pkt_addr = dma_addr;
750
751		rx_packet_len = ((qword1 & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
752				I40E_RXD_QW1_LENGTH_PBUF_SHIFT) - rxq->crc_len;
753
754		rxm->data_off = RTE_PKTMBUF_HEADROOM;
755		rte_prefetch0(RTE_PTR_ADD(rxm->buf_addr, RTE_PKTMBUF_HEADROOM));
756		rxm->nb_segs = 1;
757		rxm->next = NULL;
758		rxm->pkt_len = rx_packet_len;
759		rxm->data_len = rx_packet_len;
760		rxm->port = rxq->port_id;
761		rxm->ol_flags = 0;
762		i40e_rxd_to_vlan_tci(rxm, &rxd);
763		pkt_flags = i40e_rxd_status_to_pkt_flags(qword1);
764		pkt_flags |= i40e_rxd_error_to_pkt_flags(qword1);
765		rxm->packet_type =
766			i40e_rxd_pkt_type_mapping((uint8_t)((qword1 &
767			I40E_RXD_QW1_PTYPE_MASK) >> I40E_RXD_QW1_PTYPE_SHIFT));
768		if (pkt_flags & PKT_RX_RSS_HASH)
769			rxm->hash.rss =
770				rte_le_to_cpu_32(rxd.wb.qword0.hi_dword.rss);
771		if (pkt_flags & PKT_RX_FDIR)
772			pkt_flags |= i40e_rxd_build_fdir(&rxd, rxm);
773
774#ifdef RTE_LIBRTE_IEEE1588
775		pkt_flags |= i40e_get_iee15888_flags(rxm, qword1);
776#endif
777		rxm->ol_flags |= pkt_flags;
778
779		rx_pkts[nb_rx++] = rxm;
780	}
781	rxq->rx_tail = rx_id;
782
783	/**
784	 * If the number of free RX descriptors is greater than the RX free
785	 * threshold of the queue, advance the receive tail register of queue.
786	 * Update that register with the value of the last processed RX
787	 * descriptor minus 1.
788	 */
789	nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold);
790	if (nb_hold > rxq->rx_free_thresh) {
791		rx_id = (uint16_t) ((rx_id == 0) ?
792			(rxq->nb_rx_desc - 1) : (rx_id - 1));
793		I40E_PCI_REG_WRITE(rxq->qrx_tail, rx_id);
794		nb_hold = 0;
795	}
796	rxq->nb_rx_hold = nb_hold;
797
798	return nb_rx;
799}
800
801uint16_t
802i40e_recv_scattered_pkts(void *rx_queue,
803			 struct rte_mbuf **rx_pkts,
804			 uint16_t nb_pkts)
805{
806	struct i40e_rx_queue *rxq = rx_queue;
807	volatile union i40e_rx_desc *rx_ring = rxq->rx_ring;
808	volatile union i40e_rx_desc *rxdp;
809	union i40e_rx_desc rxd;
810	struct i40e_rx_entry *sw_ring = rxq->sw_ring;
811	struct i40e_rx_entry *rxe;
812	struct rte_mbuf *first_seg = rxq->pkt_first_seg;
813	struct rte_mbuf *last_seg = rxq->pkt_last_seg;
814	struct rte_mbuf *nmb, *rxm;
815	uint16_t rx_id = rxq->rx_tail;
816	uint16_t nb_rx = 0, nb_hold = 0, rx_packet_len;
817	uint32_t rx_status;
818	uint64_t qword1;
819	uint64_t dma_addr;
820	uint64_t pkt_flags;
821
822	while (nb_rx < nb_pkts) {
823		rxdp = &rx_ring[rx_id];
824		qword1 = rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len);
825		rx_status = (qword1 & I40E_RXD_QW1_STATUS_MASK) >>
826					I40E_RXD_QW1_STATUS_SHIFT;
827
828		/* Check the DD bit */
829		if (!(rx_status & (1 << I40E_RX_DESC_STATUS_DD_SHIFT)))
830			break;
831
832		nmb = rte_mbuf_raw_alloc(rxq->mp);
833		if (unlikely(!nmb))
834			break;
835		rxd = *rxdp;
836		nb_hold++;
837		rxe = &sw_ring[rx_id];
838		rx_id++;
839		if (rx_id == rxq->nb_rx_desc)
840			rx_id = 0;
841
842		/* Prefetch next mbuf */
843		rte_prefetch0(sw_ring[rx_id].mbuf);
844
845		/**
846		 * When next RX descriptor is on a cache line boundary,
847		 * prefetch the next 4 RX descriptors and next 8 pointers
848		 * to mbufs.
849		 */
850		if ((rx_id & 0x3) == 0) {
851			rte_prefetch0(&rx_ring[rx_id]);
852			rte_prefetch0(&sw_ring[rx_id]);
853		}
854
855		rxm = rxe->mbuf;
856		rxe->mbuf = nmb;
857		dma_addr =
858			rte_cpu_to_le_64(rte_mbuf_data_dma_addr_default(nmb));
859
860		/* Set data buffer address and data length of the mbuf */
861		rxdp->read.hdr_addr = 0;
862		rxdp->read.pkt_addr = dma_addr;
863		rx_packet_len = (qword1 & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
864					I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
865		rxm->data_len = rx_packet_len;
866		rxm->data_off = RTE_PKTMBUF_HEADROOM;
867
868		/**
869		 * If this is the first buffer of the received packet, set the
870		 * pointer to the first mbuf of the packet and initialize its
871		 * context. Otherwise, update the total length and the number
872		 * of segments of the current scattered packet, and update the
873		 * pointer to the last mbuf of the current packet.
874		 */
875		if (!first_seg) {
876			first_seg = rxm;
877			first_seg->nb_segs = 1;
878			first_seg->pkt_len = rx_packet_len;
879		} else {
880			first_seg->pkt_len =
881				(uint16_t)(first_seg->pkt_len +
882						rx_packet_len);
883			first_seg->nb_segs++;
884			last_seg->next = rxm;
885		}
886
887		/**
888		 * If this is not the last buffer of the received packet,
889		 * update the pointer to the last mbuf of the current scattered
890		 * packet and continue to parse the RX ring.
891		 */
892		if (!(rx_status & (1 << I40E_RX_DESC_STATUS_EOF_SHIFT))) {
893			last_seg = rxm;
894			continue;
895		}
896
897		/**
898		 * This is the last buffer of the received packet. If the CRC
899		 * is not stripped by the hardware:
900		 *  - Subtract the CRC length from the total packet length.
901		 *  - If the last buffer only contains the whole CRC or a part
902		 *  of it, free the mbuf associated to the last buffer. If part
903		 *  of the CRC is also contained in the previous mbuf, subtract
904		 *  the length of that CRC part from the data length of the
905		 *  previous mbuf.
906		 */
907		rxm->next = NULL;
908		if (unlikely(rxq->crc_len > 0)) {
909			first_seg->pkt_len -= ETHER_CRC_LEN;
910			if (rx_packet_len <= ETHER_CRC_LEN) {
911				rte_pktmbuf_free_seg(rxm);
912				first_seg->nb_segs--;
913				last_seg->data_len =
914					(uint16_t)(last_seg->data_len -
915					(ETHER_CRC_LEN - rx_packet_len));
916				last_seg->next = NULL;
917			} else
918				rxm->data_len = (uint16_t)(rx_packet_len -
919								ETHER_CRC_LEN);
920		}
921
922		first_seg->port = rxq->port_id;
923		first_seg->ol_flags = 0;
924		i40e_rxd_to_vlan_tci(first_seg, &rxd);
925		pkt_flags = i40e_rxd_status_to_pkt_flags(qword1);
926		pkt_flags |= i40e_rxd_error_to_pkt_flags(qword1);
927		first_seg->packet_type =
928			i40e_rxd_pkt_type_mapping((uint8_t)((qword1 &
929			I40E_RXD_QW1_PTYPE_MASK) >> I40E_RXD_QW1_PTYPE_SHIFT));
930		if (pkt_flags & PKT_RX_RSS_HASH)
931			first_seg->hash.rss =
932				rte_le_to_cpu_32(rxd.wb.qword0.hi_dword.rss);
933		if (pkt_flags & PKT_RX_FDIR)
934			pkt_flags |= i40e_rxd_build_fdir(&rxd, first_seg);
935
936#ifdef RTE_LIBRTE_IEEE1588
937		pkt_flags |= i40e_get_iee15888_flags(first_seg, qword1);
938#endif
939		first_seg->ol_flags |= pkt_flags;
940
941		/* Prefetch data of first segment, if configured to do so. */
942		rte_prefetch0(RTE_PTR_ADD(first_seg->buf_addr,
943			first_seg->data_off));
944		rx_pkts[nb_rx++] = first_seg;
945		first_seg = NULL;
946	}
947
948	/* Record index of the next RX descriptor to probe. */
949	rxq->rx_tail = rx_id;
950	rxq->pkt_first_seg = first_seg;
951	rxq->pkt_last_seg = last_seg;
952
953	/**
954	 * If the number of free RX descriptors is greater than the RX free
955	 * threshold of the queue, advance the Receive Descriptor Tail (RDT)
956	 * register. Update the RDT with the value of the last processed RX
957	 * descriptor minus 1, to guarantee that the RDT register is never
958	 * equal to the RDH register, which creates a "full" ring situtation
959	 * from the hardware point of view.
960	 */
961	nb_hold = (uint16_t)(nb_hold + rxq->nb_rx_hold);
962	if (nb_hold > rxq->rx_free_thresh) {
963		rx_id = (uint16_t)(rx_id == 0 ?
964			(rxq->nb_rx_desc - 1) : (rx_id - 1));
965		I40E_PCI_REG_WRITE(rxq->qrx_tail, rx_id);
966		nb_hold = 0;
967	}
968	rxq->nb_rx_hold = nb_hold;
969
970	return nb_rx;
971}
972
973/* Check if the context descriptor is needed for TX offloading */
974static inline uint16_t
975i40e_calc_context_desc(uint64_t flags)
976{
977	static uint64_t mask = PKT_TX_OUTER_IP_CKSUM |
978		PKT_TX_TCP_SEG |
979		PKT_TX_QINQ_PKT |
980		PKT_TX_TUNNEL_MASK;
981
982#ifdef RTE_LIBRTE_IEEE1588
983	mask |= PKT_TX_IEEE1588_TMST;
984#endif
985
986	return (flags & mask) ? 1 : 0;
987}
988
989/* set i40e TSO context descriptor */
990static inline uint64_t
991i40e_set_tso_ctx(struct rte_mbuf *mbuf, union i40e_tx_offload tx_offload)
992{
993	uint64_t ctx_desc = 0;
994	uint32_t cd_cmd, hdr_len, cd_tso_len;
995
996	if (!tx_offload.l4_len) {
997		PMD_DRV_LOG(DEBUG, "L4 length set to 0");
998		return ctx_desc;
999	}
1000
1001	/**
1002	 * in case of non tunneling packet, the outer_l2_len and
1003	 * outer_l3_len must be 0.
1004	 */
1005	hdr_len = tx_offload.outer_l2_len +
1006		tx_offload.outer_l3_len +
1007		tx_offload.l2_len +
1008		tx_offload.l3_len +
1009		tx_offload.l4_len;
1010
1011	cd_cmd = I40E_TX_CTX_DESC_TSO;
1012	cd_tso_len = mbuf->pkt_len - hdr_len;
1013	ctx_desc |= ((uint64_t)cd_cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) |
1014		((uint64_t)cd_tso_len <<
1015		 I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) |
1016		((uint64_t)mbuf->tso_segsz <<
1017		 I40E_TXD_CTX_QW1_MSS_SHIFT);
1018
1019	return ctx_desc;
1020}
1021
1022uint16_t
1023i40e_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
1024{
1025	struct i40e_tx_queue *txq;
1026	struct i40e_tx_entry *sw_ring;
1027	struct i40e_tx_entry *txe, *txn;
1028	volatile struct i40e_tx_desc *txd;
1029	volatile struct i40e_tx_desc *txr;
1030	struct rte_mbuf *tx_pkt;
1031	struct rte_mbuf *m_seg;
1032	uint32_t cd_tunneling_params;
1033	uint16_t tx_id;
1034	uint16_t nb_tx;
1035	uint32_t td_cmd;
1036	uint32_t td_offset;
1037	uint32_t tx_flags;
1038	uint32_t td_tag;
1039	uint64_t ol_flags;
1040	uint16_t nb_used;
1041	uint16_t nb_ctx;
1042	uint16_t tx_last;
1043	uint16_t slen;
1044	uint64_t buf_dma_addr;
1045	union i40e_tx_offload tx_offload = {0};
1046
1047	txq = tx_queue;
1048	sw_ring = txq->sw_ring;
1049	txr = txq->tx_ring;
1050	tx_id = txq->tx_tail;
1051	txe = &sw_ring[tx_id];
1052
1053	/* Check if the descriptor ring needs to be cleaned. */
1054	if (txq->nb_tx_free < txq->tx_free_thresh)
1055		i40e_xmit_cleanup(txq);
1056
1057	for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
1058		td_cmd = 0;
1059		td_tag = 0;
1060		td_offset = 0;
1061		tx_flags = 0;
1062
1063		tx_pkt = *tx_pkts++;
1064		RTE_MBUF_PREFETCH_TO_FREE(txe->mbuf);
1065
1066		ol_flags = tx_pkt->ol_flags;
1067		tx_offload.l2_len = tx_pkt->l2_len;
1068		tx_offload.l3_len = tx_pkt->l3_len;
1069		tx_offload.outer_l2_len = tx_pkt->outer_l2_len;
1070		tx_offload.outer_l3_len = tx_pkt->outer_l3_len;
1071		tx_offload.l4_len = tx_pkt->l4_len;
1072		tx_offload.tso_segsz = tx_pkt->tso_segsz;
1073
1074		/* Calculate the number of context descriptors needed. */
1075		nb_ctx = i40e_calc_context_desc(ol_flags);
1076
1077		/**
1078		 * The number of descriptors that must be allocated for
1079		 * a packet equals to the number of the segments of that
1080		 * packet plus 1 context descriptor if needed.
1081		 */
1082		nb_used = (uint16_t)(tx_pkt->nb_segs + nb_ctx);
1083		tx_last = (uint16_t)(tx_id + nb_used - 1);
1084
1085		/* Circular ring */
1086		if (tx_last >= txq->nb_tx_desc)
1087			tx_last = (uint16_t)(tx_last - txq->nb_tx_desc);
1088
1089		if (nb_used > txq->nb_tx_free) {
1090			if (i40e_xmit_cleanup(txq) != 0) {
1091				if (nb_tx == 0)
1092					return 0;
1093				goto end_of_tx;
1094			}
1095			if (unlikely(nb_used > txq->tx_rs_thresh)) {
1096				while (nb_used > txq->nb_tx_free) {
1097					if (i40e_xmit_cleanup(txq) != 0) {
1098						if (nb_tx == 0)
1099							return 0;
1100						goto end_of_tx;
1101					}
1102				}
1103			}
1104		}
1105
1106		/* Descriptor based VLAN insertion */
1107		if (ol_flags & (PKT_TX_VLAN_PKT | PKT_TX_QINQ_PKT)) {
1108			tx_flags |= tx_pkt->vlan_tci <<
1109				I40E_TX_FLAG_L2TAG1_SHIFT;
1110			tx_flags |= I40E_TX_FLAG_INSERT_VLAN;
1111			td_cmd |= I40E_TX_DESC_CMD_IL2TAG1;
1112			td_tag = (tx_flags & I40E_TX_FLAG_L2TAG1_MASK) >>
1113						I40E_TX_FLAG_L2TAG1_SHIFT;
1114		}
1115
1116		/* Always enable CRC offload insertion */
1117		td_cmd |= I40E_TX_DESC_CMD_ICRC;
1118
1119		/* Fill in tunneling parameters if necessary */
1120		cd_tunneling_params = 0;
1121		if (ol_flags & PKT_TX_TUNNEL_MASK)
1122			i40e_parse_tunneling_params(ol_flags, tx_offload,
1123						    &cd_tunneling_params);
1124		/* Enable checksum offloading */
1125		if (ol_flags & I40E_TX_CKSUM_OFFLOAD_MASK)
1126			i40e_txd_enable_checksum(ol_flags, &td_cmd,
1127						 &td_offset, tx_offload);
1128
1129		if (nb_ctx) {
1130			/* Setup TX context descriptor if required */
1131			volatile struct i40e_tx_context_desc *ctx_txd =
1132				(volatile struct i40e_tx_context_desc *)\
1133							&txr[tx_id];
1134			uint16_t cd_l2tag2 = 0;
1135			uint64_t cd_type_cmd_tso_mss =
1136				I40E_TX_DESC_DTYPE_CONTEXT;
1137
1138			txn = &sw_ring[txe->next_id];
1139			RTE_MBUF_PREFETCH_TO_FREE(txn->mbuf);
1140			if (txe->mbuf != NULL) {
1141				rte_pktmbuf_free_seg(txe->mbuf);
1142				txe->mbuf = NULL;
1143			}
1144
1145			/* TSO enabled means no timestamp */
1146			if (ol_flags & PKT_TX_TCP_SEG)
1147				cd_type_cmd_tso_mss |=
1148					i40e_set_tso_ctx(tx_pkt, tx_offload);
1149			else {
1150#ifdef RTE_LIBRTE_IEEE1588
1151				if (ol_flags & PKT_TX_IEEE1588_TMST)
1152					cd_type_cmd_tso_mss |=
1153						((uint64_t)I40E_TX_CTX_DESC_TSYN <<
1154						 I40E_TXD_CTX_QW1_CMD_SHIFT);
1155#endif
1156			}
1157
1158			ctx_txd->tunneling_params =
1159				rte_cpu_to_le_32(cd_tunneling_params);
1160			if (ol_flags & PKT_TX_QINQ_PKT) {
1161				cd_l2tag2 = tx_pkt->vlan_tci_outer;
1162				cd_type_cmd_tso_mss |=
1163					((uint64_t)I40E_TX_CTX_DESC_IL2TAG2 <<
1164						I40E_TXD_CTX_QW1_CMD_SHIFT);
1165			}
1166			ctx_txd->l2tag2 = rte_cpu_to_le_16(cd_l2tag2);
1167			ctx_txd->type_cmd_tso_mss =
1168				rte_cpu_to_le_64(cd_type_cmd_tso_mss);
1169
1170			PMD_TX_LOG(DEBUG, "mbuf: %p, TCD[%u]:\n"
1171				"tunneling_params: %#x;\n"
1172				"l2tag2: %#hx;\n"
1173				"rsvd: %#hx;\n"
1174				"type_cmd_tso_mss: %#"PRIx64";\n",
1175				tx_pkt, tx_id,
1176				ctx_txd->tunneling_params,
1177				ctx_txd->l2tag2,
1178				ctx_txd->rsvd,
1179				ctx_txd->type_cmd_tso_mss);
1180
1181			txe->last_id = tx_last;
1182			tx_id = txe->next_id;
1183			txe = txn;
1184		}
1185
1186		m_seg = tx_pkt;
1187		do {
1188			txd = &txr[tx_id];
1189			txn = &sw_ring[txe->next_id];
1190
1191			if (txe->mbuf)
1192				rte_pktmbuf_free_seg(txe->mbuf);
1193			txe->mbuf = m_seg;
1194
1195			/* Setup TX Descriptor */
1196			slen = m_seg->data_len;
1197			buf_dma_addr = rte_mbuf_data_dma_addr(m_seg);
1198
1199			PMD_TX_LOG(DEBUG, "mbuf: %p, TDD[%u]:\n"
1200				"buf_dma_addr: %#"PRIx64";\n"
1201				"td_cmd: %#x;\n"
1202				"td_offset: %#x;\n"
1203				"td_len: %u;\n"
1204				"td_tag: %#x;\n",
1205				tx_pkt, tx_id, buf_dma_addr,
1206				td_cmd, td_offset, slen, td_tag);
1207
1208			txd->buffer_addr = rte_cpu_to_le_64(buf_dma_addr);
1209			txd->cmd_type_offset_bsz = i40e_build_ctob(td_cmd,
1210						td_offset, slen, td_tag);
1211			txe->last_id = tx_last;
1212			tx_id = txe->next_id;
1213			txe = txn;
1214			m_seg = m_seg->next;
1215		} while (m_seg != NULL);
1216
1217		/* The last packet data descriptor needs End Of Packet (EOP) */
1218		td_cmd |= I40E_TX_DESC_CMD_EOP;
1219		txq->nb_tx_used = (uint16_t)(txq->nb_tx_used + nb_used);
1220		txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_used);
1221
1222		if (txq->nb_tx_used >= txq->tx_rs_thresh) {
1223			PMD_TX_FREE_LOG(DEBUG,
1224					"Setting RS bit on TXD id="
1225					"%4u (port=%d queue=%d)",
1226					tx_last, txq->port_id, txq->queue_id);
1227
1228			td_cmd |= I40E_TX_DESC_CMD_RS;
1229
1230			/* Update txq RS bit counters */
1231			txq->nb_tx_used = 0;
1232		}
1233
1234		txd->cmd_type_offset_bsz |=
1235			rte_cpu_to_le_64(((uint64_t)td_cmd) <<
1236					I40E_TXD_QW1_CMD_SHIFT);
1237	}
1238
1239end_of_tx:
1240	rte_wmb();
1241
1242	PMD_TX_LOG(DEBUG, "port_id=%u queue_id=%u tx_tail=%u nb_tx=%u",
1243		   (unsigned) txq->port_id, (unsigned) txq->queue_id,
1244		   (unsigned) tx_id, (unsigned) nb_tx);
1245
1246	I40E_PCI_REG_WRITE_RELAXED(txq->qtx_tail, tx_id);
1247	txq->tx_tail = tx_id;
1248
1249	return nb_tx;
1250}
1251
1252static inline int __attribute__((always_inline))
1253i40e_tx_free_bufs(struct i40e_tx_queue *txq)
1254{
1255	struct i40e_tx_entry *txep;
1256	uint16_t i;
1257
1258	if ((txq->tx_ring[txq->tx_next_dd].cmd_type_offset_bsz &
1259			rte_cpu_to_le_64(I40E_TXD_QW1_DTYPE_MASK)) !=
1260			rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE))
1261		return 0;
1262
1263	txep = &(txq->sw_ring[txq->tx_next_dd - (txq->tx_rs_thresh - 1)]);
1264
1265	for (i = 0; i < txq->tx_rs_thresh; i++)
1266		rte_prefetch0((txep + i)->mbuf);
1267
1268	if (txq->txq_flags & (uint32_t)ETH_TXQ_FLAGS_NOREFCOUNT) {
1269		for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) {
1270			rte_mempool_put(txep->mbuf->pool, txep->mbuf);
1271			txep->mbuf = NULL;
1272		}
1273	} else {
1274		for (i = 0; i < txq->tx_rs_thresh; ++i, ++txep) {
1275			rte_pktmbuf_free_seg(txep->mbuf);
1276			txep->mbuf = NULL;
1277		}
1278	}
1279
1280	txq->nb_tx_free = (uint16_t)(txq->nb_tx_free + txq->tx_rs_thresh);
1281	txq->tx_next_dd = (uint16_t)(txq->tx_next_dd + txq->tx_rs_thresh);
1282	if (txq->tx_next_dd >= txq->nb_tx_desc)
1283		txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
1284
1285	return txq->tx_rs_thresh;
1286}
1287
1288/* Populate 4 descriptors with data from 4 mbufs */
1289static inline void
1290tx4(volatile struct i40e_tx_desc *txdp, struct rte_mbuf **pkts)
1291{
1292	uint64_t dma_addr;
1293	uint32_t i;
1294
1295	for (i = 0; i < 4; i++, txdp++, pkts++) {
1296		dma_addr = rte_mbuf_data_dma_addr(*pkts);
1297		txdp->buffer_addr = rte_cpu_to_le_64(dma_addr);
1298		txdp->cmd_type_offset_bsz =
1299			i40e_build_ctob((uint32_t)I40E_TD_CMD, 0,
1300					(*pkts)->data_len, 0);
1301	}
1302}
1303
1304/* Populate 1 descriptor with data from 1 mbuf */
1305static inline void
1306tx1(volatile struct i40e_tx_desc *txdp, struct rte_mbuf **pkts)
1307{
1308	uint64_t dma_addr;
1309
1310	dma_addr = rte_mbuf_data_dma_addr(*pkts);
1311	txdp->buffer_addr = rte_cpu_to_le_64(dma_addr);
1312	txdp->cmd_type_offset_bsz =
1313		i40e_build_ctob((uint32_t)I40E_TD_CMD, 0,
1314				(*pkts)->data_len, 0);
1315}
1316
1317/* Fill hardware descriptor ring with mbuf data */
1318static inline void
1319i40e_tx_fill_hw_ring(struct i40e_tx_queue *txq,
1320		     struct rte_mbuf **pkts,
1321		     uint16_t nb_pkts)
1322{
1323	volatile struct i40e_tx_desc *txdp = &(txq->tx_ring[txq->tx_tail]);
1324	struct i40e_tx_entry *txep = &(txq->sw_ring[txq->tx_tail]);
1325	const int N_PER_LOOP = 4;
1326	const int N_PER_LOOP_MASK = N_PER_LOOP - 1;
1327	int mainpart, leftover;
1328	int i, j;
1329
1330	mainpart = (nb_pkts & ((uint32_t) ~N_PER_LOOP_MASK));
1331	leftover = (nb_pkts & ((uint32_t)  N_PER_LOOP_MASK));
1332	for (i = 0; i < mainpart; i += N_PER_LOOP) {
1333		for (j = 0; j < N_PER_LOOP; ++j) {
1334			(txep + i + j)->mbuf = *(pkts + i + j);
1335		}
1336		tx4(txdp + i, pkts + i);
1337	}
1338	if (unlikely(leftover > 0)) {
1339		for (i = 0; i < leftover; ++i) {
1340			(txep + mainpart + i)->mbuf = *(pkts + mainpart + i);
1341			tx1(txdp + mainpart + i, pkts + mainpart + i);
1342		}
1343	}
1344}
1345
1346static inline uint16_t
1347tx_xmit_pkts(struct i40e_tx_queue *txq,
1348	     struct rte_mbuf **tx_pkts,
1349	     uint16_t nb_pkts)
1350{
1351	volatile struct i40e_tx_desc *txr = txq->tx_ring;
1352	uint16_t n = 0;
1353
1354	/**
1355	 * Begin scanning the H/W ring for done descriptors when the number
1356	 * of available descriptors drops below tx_free_thresh. For each done
1357	 * descriptor, free the associated buffer.
1358	 */
1359	if (txq->nb_tx_free < txq->tx_free_thresh)
1360		i40e_tx_free_bufs(txq);
1361
1362	/* Use available descriptor only */
1363	nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts);
1364	if (unlikely(!nb_pkts))
1365		return 0;
1366
1367	txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts);
1368	if ((txq->tx_tail + nb_pkts) > txq->nb_tx_desc) {
1369		n = (uint16_t)(txq->nb_tx_desc - txq->tx_tail);
1370		i40e_tx_fill_hw_ring(txq, tx_pkts, n);
1371		txr[txq->tx_next_rs].cmd_type_offset_bsz |=
1372			rte_cpu_to_le_64(((uint64_t)I40E_TX_DESC_CMD_RS) <<
1373						I40E_TXD_QW1_CMD_SHIFT);
1374		txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
1375		txq->tx_tail = 0;
1376	}
1377
1378	/* Fill hardware descriptor ring with mbuf data */
1379	i40e_tx_fill_hw_ring(txq, tx_pkts + n, (uint16_t)(nb_pkts - n));
1380	txq->tx_tail = (uint16_t)(txq->tx_tail + (nb_pkts - n));
1381
1382	/* Determin if RS bit needs to be set */
1383	if (txq->tx_tail > txq->tx_next_rs) {
1384		txr[txq->tx_next_rs].cmd_type_offset_bsz |=
1385			rte_cpu_to_le_64(((uint64_t)I40E_TX_DESC_CMD_RS) <<
1386						I40E_TXD_QW1_CMD_SHIFT);
1387		txq->tx_next_rs =
1388			(uint16_t)(txq->tx_next_rs + txq->tx_rs_thresh);
1389		if (txq->tx_next_rs >= txq->nb_tx_desc)
1390			txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
1391	}
1392
1393	if (txq->tx_tail >= txq->nb_tx_desc)
1394		txq->tx_tail = 0;
1395
1396	/* Update the tx tail register */
1397	rte_wmb();
1398	I40E_PCI_REG_WRITE_RELAXED(txq->qtx_tail, txq->tx_tail);
1399
1400	return nb_pkts;
1401}
1402
1403static uint16_t
1404i40e_xmit_pkts_simple(void *tx_queue,
1405		      struct rte_mbuf **tx_pkts,
1406		      uint16_t nb_pkts)
1407{
1408	uint16_t nb_tx = 0;
1409
1410	if (likely(nb_pkts <= I40E_TX_MAX_BURST))
1411		return tx_xmit_pkts((struct i40e_tx_queue *)tx_queue,
1412						tx_pkts, nb_pkts);
1413
1414	while (nb_pkts) {
1415		uint16_t ret, num = (uint16_t)RTE_MIN(nb_pkts,
1416						I40E_TX_MAX_BURST);
1417
1418		ret = tx_xmit_pkts((struct i40e_tx_queue *)tx_queue,
1419						&tx_pkts[nb_tx], num);
1420		nb_tx = (uint16_t)(nb_tx + ret);
1421		nb_pkts = (uint16_t)(nb_pkts - ret);
1422		if (ret < num)
1423			break;
1424	}
1425
1426	return nb_tx;
1427}
1428
1429/*********************************************************************
1430 *
1431 *  TX prep functions
1432 *
1433 **********************************************************************/
1434uint16_t
1435i40e_prep_pkts(__rte_unused void *tx_queue, struct rte_mbuf **tx_pkts,
1436		uint16_t nb_pkts)
1437{
1438	int i, ret;
1439	uint64_t ol_flags;
1440	struct rte_mbuf *m;
1441
1442	for (i = 0; i < nb_pkts; i++) {
1443		m = tx_pkts[i];
1444		ol_flags = m->ol_flags;
1445
1446		/**
1447		 * m->nb_segs is uint8_t, so nb_segs is always less than
1448		 * I40E_TX_MAX_SEG.
1449		 * We check only a condition for nb_segs > I40E_TX_MAX_MTU_SEG.
1450		 */
1451		if (!(ol_flags & PKT_TX_TCP_SEG)) {
1452			if (m->nb_segs > I40E_TX_MAX_MTU_SEG) {
1453				rte_errno = -EINVAL;
1454				return i;
1455			}
1456		} else if ((m->tso_segsz < I40E_MIN_TSO_MSS) ||
1457				(m->tso_segsz > I40E_MAX_TSO_MSS)) {
1458			/* MSS outside the range (256B - 9674B) are considered
1459			 * malicious
1460			 */
1461			rte_errno = -EINVAL;
1462			return i;
1463		}
1464
1465		if (ol_flags & I40E_TX_OFFLOAD_NOTSUP_MASK) {
1466			rte_errno = -ENOTSUP;
1467			return i;
1468		}
1469
1470#ifdef RTE_LIBRTE_ETHDEV_DEBUG
1471		ret = rte_validate_tx_offload(m);
1472		if (ret != 0) {
1473			rte_errno = ret;
1474			return i;
1475		}
1476#endif
1477		ret = rte_net_intel_cksum_prepare(m);
1478		if (ret != 0) {
1479			rte_errno = ret;
1480			return i;
1481		}
1482	}
1483	return i;
1484}
1485
1486/*
1487 * Find the VSI the queue belongs to. 'queue_idx' is the queue index
1488 * application used, which assume having sequential ones. But from driver's
1489 * perspective, it's different. For example, q0 belongs to FDIR VSI, q1-q64
1490 * to MAIN VSI, , q65-96 to SRIOV VSIs, q97-128 to VMDQ VSIs. For application
1491 * running on host, q1-64 and q97-128 can be used, total 96 queues. They can
1492 * use queue_idx from 0 to 95 to access queues, while real queue would be
1493 * different. This function will do a queue mapping to find VSI the queue
1494 * belongs to.
1495 */
1496static struct i40e_vsi*
1497i40e_pf_get_vsi_by_qindex(struct i40e_pf *pf, uint16_t queue_idx)
1498{
1499	/* the queue in MAIN VSI range */
1500	if (queue_idx < pf->main_vsi->nb_qps)
1501		return pf->main_vsi;
1502
1503	queue_idx -= pf->main_vsi->nb_qps;
1504
1505	/* queue_idx is greater than VMDQ VSIs range */
1506	if (queue_idx > pf->nb_cfg_vmdq_vsi * pf->vmdq_nb_qps - 1) {
1507		PMD_INIT_LOG(ERR, "queue_idx out of range. VMDQ configured?");
1508		return NULL;
1509	}
1510
1511	return pf->vmdq[queue_idx / pf->vmdq_nb_qps].vsi;
1512}
1513
1514static uint16_t
1515i40e_get_queue_offset_by_qindex(struct i40e_pf *pf, uint16_t queue_idx)
1516{
1517	/* the queue in MAIN VSI range */
1518	if (queue_idx < pf->main_vsi->nb_qps)
1519		return queue_idx;
1520
1521	/* It's VMDQ queues */
1522	queue_idx -= pf->main_vsi->nb_qps;
1523
1524	if (pf->nb_cfg_vmdq_vsi)
1525		return queue_idx % pf->vmdq_nb_qps;
1526	else {
1527		PMD_INIT_LOG(ERR, "Fail to get queue offset");
1528		return (uint16_t)(-1);
1529	}
1530}
1531
1532int
1533i40e_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1534{
1535	struct i40e_rx_queue *rxq;
1536	int err = -1;
1537	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1538
1539	PMD_INIT_FUNC_TRACE();
1540
1541	if (rx_queue_id < dev->data->nb_rx_queues) {
1542		rxq = dev->data->rx_queues[rx_queue_id];
1543
1544		err = i40e_alloc_rx_queue_mbufs(rxq);
1545		if (err) {
1546			PMD_DRV_LOG(ERR, "Failed to allocate RX queue mbuf");
1547			return err;
1548		}
1549
1550		rte_wmb();
1551
1552		/* Init the RX tail regieter. */
1553		I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
1554
1555		err = i40e_switch_rx_queue(hw, rxq->reg_idx, TRUE);
1556
1557		if (err) {
1558			PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on",
1559				    rx_queue_id);
1560
1561			i40e_rx_queue_release_mbufs(rxq);
1562			i40e_reset_rx_queue(rxq);
1563		} else
1564			dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
1565	}
1566
1567	return err;
1568}
1569
1570int
1571i40e_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1572{
1573	struct i40e_rx_queue *rxq;
1574	int err;
1575	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1576
1577	if (rx_queue_id < dev->data->nb_rx_queues) {
1578		rxq = dev->data->rx_queues[rx_queue_id];
1579
1580		/*
1581		* rx_queue_id is queue id aplication refers to, while
1582		* rxq->reg_idx is the real queue index.
1583		*/
1584		err = i40e_switch_rx_queue(hw, rxq->reg_idx, FALSE);
1585
1586		if (err) {
1587			PMD_DRV_LOG(ERR, "Failed to switch RX queue %u off",
1588				    rx_queue_id);
1589			return err;
1590		}
1591		i40e_rx_queue_release_mbufs(rxq);
1592		i40e_reset_rx_queue(rxq);
1593		dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
1594	}
1595
1596	return 0;
1597}
1598
1599int
1600i40e_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
1601{
1602	int err = -1;
1603	struct i40e_tx_queue *txq;
1604	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1605
1606	PMD_INIT_FUNC_TRACE();
1607
1608	if (tx_queue_id < dev->data->nb_tx_queues) {
1609		txq = dev->data->tx_queues[tx_queue_id];
1610
1611		/*
1612		* tx_queue_id is queue id aplication refers to, while
1613		* rxq->reg_idx is the real queue index.
1614		*/
1615		err = i40e_switch_tx_queue(hw, txq->reg_idx, TRUE);
1616		if (err)
1617			PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on",
1618				    tx_queue_id);
1619		else
1620			dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
1621	}
1622
1623	return err;
1624}
1625
1626int
1627i40e_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
1628{
1629	struct i40e_tx_queue *txq;
1630	int err;
1631	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1632
1633	if (tx_queue_id < dev->data->nb_tx_queues) {
1634		txq = dev->data->tx_queues[tx_queue_id];
1635
1636		/*
1637		* tx_queue_id is queue id aplication refers to, while
1638		* txq->reg_idx is the real queue index.
1639		*/
1640		err = i40e_switch_tx_queue(hw, txq->reg_idx, FALSE);
1641
1642		if (err) {
1643			PMD_DRV_LOG(ERR, "Failed to switch TX queue %u of",
1644				    tx_queue_id);
1645			return err;
1646		}
1647
1648		i40e_tx_queue_release_mbufs(txq);
1649		i40e_reset_tx_queue(txq);
1650		dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
1651	}
1652
1653	return 0;
1654}
1655
1656const uint32_t *
1657i40e_dev_supported_ptypes_get(struct rte_eth_dev *dev)
1658{
1659	static const uint32_t ptypes[] = {
1660		/* refers to i40e_rxd_pkt_type_mapping() */
1661		RTE_PTYPE_L2_ETHER,
1662		RTE_PTYPE_L2_ETHER_TIMESYNC,
1663		RTE_PTYPE_L2_ETHER_LLDP,
1664		RTE_PTYPE_L2_ETHER_ARP,
1665		RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
1666		RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
1667		RTE_PTYPE_L4_FRAG,
1668		RTE_PTYPE_L4_ICMP,
1669		RTE_PTYPE_L4_NONFRAG,
1670		RTE_PTYPE_L4_SCTP,
1671		RTE_PTYPE_L4_TCP,
1672		RTE_PTYPE_L4_UDP,
1673		RTE_PTYPE_TUNNEL_GRENAT,
1674		RTE_PTYPE_TUNNEL_IP,
1675		RTE_PTYPE_INNER_L2_ETHER,
1676		RTE_PTYPE_INNER_L2_ETHER_VLAN,
1677		RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN,
1678		RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN,
1679		RTE_PTYPE_INNER_L4_FRAG,
1680		RTE_PTYPE_INNER_L4_ICMP,
1681		RTE_PTYPE_INNER_L4_NONFRAG,
1682		RTE_PTYPE_INNER_L4_SCTP,
1683		RTE_PTYPE_INNER_L4_TCP,
1684		RTE_PTYPE_INNER_L4_UDP,
1685		RTE_PTYPE_UNKNOWN
1686	};
1687
1688	if (dev->rx_pkt_burst == i40e_recv_pkts ||
1689#ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC
1690	    dev->rx_pkt_burst == i40e_recv_pkts_bulk_alloc ||
1691#endif
1692	    dev->rx_pkt_burst == i40e_recv_scattered_pkts ||
1693	    dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec ||
1694	    dev->rx_pkt_burst == i40e_recv_pkts_vec)
1695		return ptypes;
1696	return NULL;
1697}
1698
1699int
1700i40e_dev_rx_queue_setup(struct rte_eth_dev *dev,
1701			uint16_t queue_idx,
1702			uint16_t nb_desc,
1703			unsigned int socket_id,
1704			const struct rte_eth_rxconf *rx_conf,
1705			struct rte_mempool *mp)
1706{
1707	struct i40e_vsi *vsi;
1708	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1709	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1710	struct i40e_adapter *ad =
1711		I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1712	struct i40e_rx_queue *rxq;
1713	const struct rte_memzone *rz;
1714	uint32_t ring_size;
1715	uint16_t len, i;
1716	uint16_t base, bsf, tc_mapping;
1717	int use_def_burst_func = 1;
1718
1719#define TREX_PATCH_LOW_LATENCY
1720#ifdef TREX_PATCH_LOW_LATENCY
1721    int is_vf = 0;
1722#endif
1723
1724	if (hw->mac.type == I40E_MAC_VF || hw->mac.type == I40E_MAC_X722_VF) {
1725		struct i40e_vf *vf =
1726			I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1727		vsi = &vf->vsi;
1728#ifdef TREX_PATCH_LOW_LATENCY
1729        is_vf = 1;
1730#endif
1731	} else {
1732		vsi = i40e_pf_get_vsi_by_qindex(pf, queue_idx);
1733    }
1734
1735	if (vsi == NULL) {
1736		PMD_DRV_LOG(ERR, "VSI not available or queue "
1737			    "index exceeds the maximum");
1738		return I40E_ERR_PARAM;
1739	}
1740	if (nb_desc % I40E_ALIGN_RING_DESC != 0 ||
1741			(nb_desc > I40E_MAX_RING_DESC) ||
1742			(nb_desc < I40E_MIN_RING_DESC)) {
1743		PMD_DRV_LOG(ERR, "Number (%u) of receive descriptors is "
1744			    "invalid", nb_desc);
1745		return I40E_ERR_PARAM;
1746	}
1747
1748	/* Free memory if needed */
1749	if (dev->data->rx_queues[queue_idx]) {
1750		i40e_dev_rx_queue_release(dev->data->rx_queues[queue_idx]);
1751		dev->data->rx_queues[queue_idx] = NULL;
1752	}
1753
1754	/* Allocate the rx queue data structure */
1755	rxq = rte_zmalloc_socket("i40e rx queue",
1756				 sizeof(struct i40e_rx_queue),
1757				 RTE_CACHE_LINE_SIZE,
1758				 socket_id);
1759	if (!rxq) {
1760		PMD_DRV_LOG(ERR, "Failed to allocate memory for "
1761			    "rx queue data structure");
1762		return -ENOMEM;
1763	}
1764	rxq->mp = mp;
1765	rxq->nb_rx_desc = nb_desc;
1766	rxq->rx_free_thresh = rx_conf->rx_free_thresh;
1767	rxq->queue_id = queue_idx;
1768	if (hw->mac.type == I40E_MAC_VF || hw->mac.type == I40E_MAC_X722_VF)
1769		rxq->reg_idx = queue_idx;
1770	else /* PF device */
1771		rxq->reg_idx = vsi->base_queue +
1772			i40e_get_queue_offset_by_qindex(pf, queue_idx);
1773
1774	rxq->port_id = dev->data->port_id;
1775	rxq->crc_len = (uint8_t) ((dev->data->dev_conf.rxmode.hw_strip_crc) ?
1776							0 : ETHER_CRC_LEN);
1777	rxq->drop_en = rx_conf->rx_drop_en;
1778	rxq->vsi = vsi;
1779	rxq->rx_deferred_start = rx_conf->rx_deferred_start;
1780
1781	/* Allocate the maximun number of RX ring hardware descriptor. */
1782	len = I40E_MAX_RING_DESC;
1783
1784#ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC
1785	/**
1786	 * Allocating a little more memory because vectorized/bulk_alloc Rx
1787	 * functions doesn't check boundaries each time.
1788	 */
1789	len += RTE_PMD_I40E_RX_MAX_BURST;
1790#endif
1791
1792	ring_size = RTE_ALIGN(len * sizeof(union i40e_rx_desc),
1793			      I40E_DMA_MEM_ALIGN);
1794
1795	rz = rte_eth_dma_zone_reserve(dev, "rx_ring", queue_idx,
1796			      ring_size, I40E_RING_BASE_ALIGN, socket_id);
1797	if (!rz) {
1798		i40e_dev_rx_queue_release(rxq);
1799		PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for RX");
1800		return -ENOMEM;
1801	}
1802
1803	/* Zero all the descriptors in the ring. */
1804	memset(rz->addr, 0, ring_size);
1805
1806	rxq->rx_ring_phys_addr = rte_mem_phy2mch(rz->memseg_id, rz->phys_addr);
1807	rxq->rx_ring = (union i40e_rx_desc *)rz->addr;
1808
1809#ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC
1810	len = (uint16_t)(nb_desc + RTE_PMD_I40E_RX_MAX_BURST);
1811#else
1812	len = nb_desc;
1813#endif
1814
1815	/* Allocate the software ring. */
1816	rxq->sw_ring =
1817		rte_zmalloc_socket("i40e rx sw ring",
1818				   sizeof(struct i40e_rx_entry) * len,
1819				   RTE_CACHE_LINE_SIZE,
1820				   socket_id);
1821	if (!rxq->sw_ring) {
1822		i40e_dev_rx_queue_release(rxq);
1823		PMD_DRV_LOG(ERR, "Failed to allocate memory for SW ring");
1824		return -ENOMEM;
1825	}
1826
1827	i40e_reset_rx_queue(rxq);
1828	rxq->q_set = TRUE;
1829	dev->data->rx_queues[queue_idx] = rxq;
1830
1831	use_def_burst_func = check_rx_burst_bulk_alloc_preconditions(rxq);
1832
1833	if (!use_def_burst_func) {
1834#ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC
1835		PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
1836			     "satisfied. Rx Burst Bulk Alloc function will be "
1837			     "used on port=%d, queue=%d.",
1838			     rxq->port_id, rxq->queue_id);
1839#endif /* RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC */
1840	} else {
1841		PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
1842			     "not satisfied, Scattered Rx is requested, "
1843			     "or RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC is "
1844			     "not enabled on port=%d, queue=%d.",
1845			     rxq->port_id, rxq->queue_id);
1846		ad->rx_bulk_alloc_allowed = false;
1847	}
1848
1849#ifdef TREX_PATCH_LOW_LATENCY
1850    if (! is_vf)
1851        rxq->dcb_tc =0;
1852    else // The entire for below is in the else
1853#endif
1854
1855	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
1856		if (!(vsi->enabled_tc & (1 << i)))
1857			continue;
1858		tc_mapping = rte_le_to_cpu_16(vsi->info.tc_mapping[i]);
1859		base = (tc_mapping & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) >>
1860			I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT;
1861		bsf = (tc_mapping & I40E_AQ_VSI_TC_QUE_NUMBER_MASK) >>
1862			I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT;
1863
1864		if (queue_idx >= base && queue_idx < (base + BIT(bsf)))
1865			rxq->dcb_tc = i;
1866	}
1867
1868	return 0;
1869}
1870
1871void
1872i40e_dev_rx_queue_release(void *rxq)
1873{
1874	struct i40e_rx_queue *q = (struct i40e_rx_queue *)rxq;
1875
1876	if (!q) {
1877		PMD_DRV_LOG(DEBUG, "Pointer to rxq is NULL");
1878		return;
1879	}
1880
1881	i40e_rx_queue_release_mbufs(q);
1882	rte_free(q->sw_ring);
1883	rte_free(q);
1884}
1885
1886uint32_t
1887i40e_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1888{
1889#define I40E_RXQ_SCAN_INTERVAL 4
1890	volatile union i40e_rx_desc *rxdp;
1891	struct i40e_rx_queue *rxq;
1892	uint16_t desc = 0;
1893
1894	if (unlikely(rx_queue_id >= dev->data->nb_rx_queues)) {
1895		PMD_DRV_LOG(ERR, "Invalid RX queue id %u", rx_queue_id);
1896		return 0;
1897	}
1898
1899	rxq = dev->data->rx_queues[rx_queue_id];
1900	rxdp = &(rxq->rx_ring[rxq->rx_tail]);
1901	while ((desc < rxq->nb_rx_desc) &&
1902		((rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len) &
1903		I40E_RXD_QW1_STATUS_MASK) >> I40E_RXD_QW1_STATUS_SHIFT) &
1904				(1 << I40E_RX_DESC_STATUS_DD_SHIFT)) {
1905		/**
1906		 * Check the DD bit of a rx descriptor of each 4 in a group,
1907		 * to avoid checking too frequently and downgrading performance
1908		 * too much.
1909		 */
1910		desc += I40E_RXQ_SCAN_INTERVAL;
1911		rxdp += I40E_RXQ_SCAN_INTERVAL;
1912		if (rxq->rx_tail + desc >= rxq->nb_rx_desc)
1913			rxdp = &(rxq->rx_ring[rxq->rx_tail +
1914					desc - rxq->nb_rx_desc]);
1915	}
1916
1917	return desc;
1918}
1919
1920int
1921i40e_dev_rx_descriptor_done(void *rx_queue, uint16_t offset)
1922{
1923	volatile union i40e_rx_desc *rxdp;
1924	struct i40e_rx_queue *rxq = rx_queue;
1925	uint16_t desc;
1926	int ret;
1927
1928	if (unlikely(offset >= rxq->nb_rx_desc)) {
1929		PMD_DRV_LOG(ERR, "Invalid RX queue id %u", offset);
1930		return 0;
1931	}
1932
1933	desc = rxq->rx_tail + offset;
1934	if (desc >= rxq->nb_rx_desc)
1935		desc -= rxq->nb_rx_desc;
1936
1937	rxdp = &(rxq->rx_ring[desc]);
1938
1939	ret = !!(((rte_le_to_cpu_64(rxdp->wb.qword1.status_error_len) &
1940		I40E_RXD_QW1_STATUS_MASK) >> I40E_RXD_QW1_STATUS_SHIFT) &
1941				(1 << I40E_RX_DESC_STATUS_DD_SHIFT));
1942
1943	return ret;
1944}
1945
1946int
1947i40e_dev_tx_queue_setup(struct rte_eth_dev *dev,
1948			uint16_t queue_idx,
1949			uint16_t nb_desc,
1950			unsigned int socket_id,
1951			const struct rte_eth_txconf *tx_conf)
1952{
1953	struct i40e_vsi *vsi;
1954	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1955	struct i40e_pf *pf = I40E_DEV_PRIVATE_TO_PF(dev->data->dev_private);
1956	struct i40e_tx_queue *txq;
1957	const struct rte_memzone *tz;
1958	uint32_t ring_size;
1959	uint16_t tx_rs_thresh, tx_free_thresh;
1960	uint16_t i, base, bsf, tc_mapping;
1961
1962#ifdef TREX_PATCH_LOW_LATENCY
1963    u8 low_latency = 0;
1964    int is_vf = 1;
1965#endif
1966
1967	if (hw->mac.type == I40E_MAC_VF || hw->mac.type == I40E_MAC_X722_VF) {
1968		struct i40e_vf *vf =
1969			I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1970		vsi = &vf->vsi;
1971	} else {
1972		vsi = i40e_pf_get_vsi_by_qindex(pf, queue_idx);
1973#ifdef TREX_PATCH_LOW_LATENCY
1974        if (queue_idx == pf->dev_data->nb_tx_queues-1) {
1975            low_latency = 1;
1976        }
1977        is_vf = 0;
1978#endif
1979    }
1980
1981	if (vsi == NULL) {
1982		PMD_DRV_LOG(ERR, "VSI is NULL, or queue index (%u) "
1983			    "exceeds the maximum", queue_idx);
1984		return I40E_ERR_PARAM;
1985	}
1986
1987	if (nb_desc % I40E_ALIGN_RING_DESC != 0 ||
1988			(nb_desc > I40E_MAX_RING_DESC) ||
1989			(nb_desc < I40E_MIN_RING_DESC)) {
1990		PMD_DRV_LOG(ERR, "Number (%u) of transmit descriptors is "
1991			    "invalid", nb_desc);
1992		return I40E_ERR_PARAM;
1993	}
1994
1995	/**
1996	 * The following two parameters control the setting of the RS bit on
1997	 * transmit descriptors. TX descriptors will have their RS bit set
1998	 * after txq->tx_rs_thresh descriptors have been used. The TX
1999	 * descriptor ring will be cleaned after txq->tx_free_thresh
2000	 * descriptors are used or if the number of descriptors required to
2001	 * transmit a packet is greater than the number of free TX descriptors.
2002	 *
2003	 * The following constraints must be satisfied:
2004	 *  - tx_rs_thresh must be greater than 0.
2005	 *  - tx_rs_thresh must be less than the size of the ring minus 2.
2006	 *  - tx_rs_thresh must be less than or equal to tx_free_thresh.
2007	 *  - tx_rs_thresh must be a divisor of the ring size.
2008	 *  - tx_free_thresh must be greater than 0.
2009	 *  - tx_free_thresh must be less than the size of the ring minus 3.
2010	 *
2011	 * One descriptor in the TX ring is used as a sentinel to avoid a H/W
2012	 * race condition, hence the maximum threshold constraints. When set
2013	 * to zero use default values.
2014	 */
2015	tx_rs_thresh = (uint16_t)((tx_conf->tx_rs_thresh) ?
2016		tx_conf->tx_rs_thresh : DEFAULT_TX_RS_THRESH);
2017	tx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ?
2018		tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH);
2019	if (tx_rs_thresh >= (nb_desc - 2)) {
2020		PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than the "
2021			     "number of TX descriptors minus 2. "
2022			     "(tx_rs_thresh=%u port=%d queue=%d)",
2023			     (unsigned int)tx_rs_thresh,
2024			     (int)dev->data->port_id,
2025			     (int)queue_idx);
2026		return I40E_ERR_PARAM;
2027	}
2028	if (tx_free_thresh >= (nb_desc - 3)) {
2029		PMD_INIT_LOG(ERR, "tx_free_thresh must be less than the "
2030			     "number of TX descriptors minus 3. "
2031			     "(tx_free_thresh=%u port=%d queue=%d)",
2032			     (unsigned int)tx_free_thresh,
2033			     (int)dev->data->port_id,
2034			     (int)queue_idx);
2035		return I40E_ERR_PARAM;
2036	}
2037	if (tx_rs_thresh > tx_free_thresh) {
2038		PMD_INIT_LOG(ERR, "tx_rs_thresh must be less than or "
2039			     "equal to tx_free_thresh. (tx_free_thresh=%u"
2040			     " tx_rs_thresh=%u port=%d queue=%d)",
2041			     (unsigned int)tx_free_thresh,
2042			     (unsigned int)tx_rs_thresh,
2043			     (int)dev->data->port_id,
2044			     (int)queue_idx);
2045		return I40E_ERR_PARAM;
2046	}
2047	if ((nb_desc % tx_rs_thresh) != 0) {
2048		PMD_INIT_LOG(ERR, "tx_rs_thresh must be a divisor of the "
2049			     "number of TX descriptors. (tx_rs_thresh=%u"
2050			     " port=%d queue=%d)",
2051			     (unsigned int)tx_rs_thresh,
2052			     (int)dev->data->port_id,
2053			     (int)queue_idx);
2054		return I40E_ERR_PARAM;
2055	}
2056	if ((tx_rs_thresh > 1) && (tx_conf->tx_thresh.wthresh != 0)) {
2057		PMD_INIT_LOG(ERR, "TX WTHRESH must be set to 0 if "
2058			     "tx_rs_thresh is greater than 1. "
2059			     "(tx_rs_thresh=%u port=%d queue=%d)",
2060			     (unsigned int)tx_rs_thresh,
2061			     (int)dev->data->port_id,
2062			     (int)queue_idx);
2063		return I40E_ERR_PARAM;
2064	}
2065
2066	/* Free memory if needed. */
2067	if (dev->data->tx_queues[queue_idx]) {
2068		i40e_dev_tx_queue_release(dev->data->tx_queues[queue_idx]);
2069		dev->data->tx_queues[queue_idx] = NULL;
2070	}
2071
2072	/* Allocate the TX queue data structure. */
2073	txq = rte_zmalloc_socket("i40e tx queue",
2074				  sizeof(struct i40e_tx_queue),
2075				  RTE_CACHE_LINE_SIZE,
2076				  socket_id);
2077	if (!txq) {
2078		PMD_DRV_LOG(ERR, "Failed to allocate memory for "
2079			    "tx queue structure");
2080		return -ENOMEM;
2081	}
2082
2083	/* Allocate TX hardware ring descriptors. */
2084	ring_size = sizeof(struct i40e_tx_desc) * I40E_MAX_RING_DESC;
2085	ring_size = RTE_ALIGN(ring_size, I40E_DMA_MEM_ALIGN);
2086	tz = rte_eth_dma_zone_reserve(dev, "tx_ring", queue_idx,
2087			      ring_size, I40E_RING_BASE_ALIGN, socket_id);
2088	if (!tz) {
2089		i40e_dev_tx_queue_release(txq);
2090		PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for TX");
2091		return -ENOMEM;
2092	}
2093
2094	txq->nb_tx_desc = nb_desc;
2095	txq->tx_rs_thresh = tx_rs_thresh;
2096	txq->tx_free_thresh = tx_free_thresh;
2097	txq->pthresh = tx_conf->tx_thresh.pthresh;
2098	txq->hthresh = tx_conf->tx_thresh.hthresh;
2099	txq->wthresh = tx_conf->tx_thresh.wthresh;
2100	txq->queue_id = queue_idx;
2101	if (hw->mac.type == I40E_MAC_VF || hw->mac.type == I40E_MAC_X722_VF)
2102		txq->reg_idx = queue_idx;
2103	else /* PF device */
2104		txq->reg_idx = vsi->base_queue +
2105			i40e_get_queue_offset_by_qindex(pf, queue_idx);
2106
2107	txq->port_id = dev->data->port_id;
2108	txq->txq_flags = tx_conf->txq_flags;
2109	txq->vsi = vsi;
2110	txq->tx_deferred_start = tx_conf->tx_deferred_start;
2111
2112	txq->tx_ring_phys_addr = rte_mem_phy2mch(tz->memseg_id, tz->phys_addr);
2113	txq->tx_ring = (struct i40e_tx_desc *)tz->addr;
2114
2115	/* Allocate software ring */
2116	txq->sw_ring =
2117		rte_zmalloc_socket("i40e tx sw ring",
2118				   sizeof(struct i40e_tx_entry) * nb_desc,
2119				   RTE_CACHE_LINE_SIZE,
2120				   socket_id);
2121	if (!txq->sw_ring) {
2122		i40e_dev_tx_queue_release(txq);
2123		PMD_DRV_LOG(ERR, "Failed to allocate memory for SW TX ring");
2124		return -ENOMEM;
2125	}
2126
2127	i40e_reset_tx_queue(txq);
2128	txq->q_set = TRUE;
2129	dev->data->tx_queues[queue_idx] = txq;
2130
2131	/* Use a simple TX queue without offloads or multi segs if possible */
2132	i40e_set_tx_function_flag(dev, txq);
2133
2134#ifdef TREX_PATCH_LOW_LATENCY
2135    if (! is_vf) {
2136        if (low_latency) {
2137            txq->dcb_tc=1;
2138        }else{
2139            txq->dcb_tc=0;
2140        }
2141    } else // The entire for below is in the else
2142#endif
2143	for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) {
2144		if (!(vsi->enabled_tc & (1 << i)))
2145			continue;
2146		tc_mapping = rte_le_to_cpu_16(vsi->info.tc_mapping[i]);
2147		base = (tc_mapping & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) >>
2148			I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT;
2149		bsf = (tc_mapping & I40E_AQ_VSI_TC_QUE_NUMBER_MASK) >>
2150			I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT;
2151
2152		if (queue_idx >= base && queue_idx < (base + BIT(bsf)))
2153			txq->dcb_tc = i;
2154	}
2155
2156	return 0;
2157}
2158
2159void
2160i40e_dev_tx_queue_release(void *txq)
2161{
2162	struct i40e_tx_queue *q = (struct i40e_tx_queue *)txq;
2163
2164	if (!q) {
2165		PMD_DRV_LOG(DEBUG, "Pointer to TX queue is NULL");
2166		return;
2167	}
2168
2169	i40e_tx_queue_release_mbufs(q);
2170	rte_free(q->sw_ring);
2171	rte_free(q);
2172}
2173
2174const struct rte_memzone *
2175i40e_memzone_reserve(const char *name, uint32_t len, int socket_id)
2176{
2177	const struct rte_memzone *mz;
2178
2179	mz = rte_memzone_lookup(name);
2180	if (mz)
2181		return mz;
2182
2183	if (rte_xen_dom0_supported())
2184		mz = rte_memzone_reserve_bounded(name, len,
2185				socket_id, 0, I40E_RING_BASE_ALIGN, RTE_PGSIZE_2M);
2186	else
2187		mz = rte_memzone_reserve_aligned(name, len,
2188				socket_id, 0, I40E_RING_BASE_ALIGN);
2189	return mz;
2190}
2191
2192void
2193i40e_rx_queue_release_mbufs(struct i40e_rx_queue *rxq)
2194{
2195	uint16_t i;
2196
2197	/* SSE Vector driver has a different way of releasing mbufs. */
2198	if (rxq->rx_using_sse) {
2199		i40e_rx_queue_release_mbufs_vec(rxq);
2200		return;
2201	}
2202
2203	if (!rxq->sw_ring) {
2204		PMD_DRV_LOG(DEBUG, "Pointer to sw_ring is NULL");
2205		return;
2206	}
2207
2208	for (i = 0; i < rxq->nb_rx_desc; i++) {
2209		if (rxq->sw_ring[i].mbuf) {
2210			rte_pktmbuf_free_seg(rxq->sw_ring[i].mbuf);
2211			rxq->sw_ring[i].mbuf = NULL;
2212		}
2213	}
2214#ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC
2215	if (rxq->rx_nb_avail == 0)
2216		return;
2217	for (i = 0; i < rxq->rx_nb_avail; i++) {
2218		struct rte_mbuf *mbuf;
2219
2220		mbuf = rxq->rx_stage[rxq->rx_next_avail + i];
2221		rte_pktmbuf_free_seg(mbuf);
2222	}
2223	rxq->rx_nb_avail = 0;
2224#endif /* RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC */
2225}
2226
2227void
2228i40e_reset_rx_queue(struct i40e_rx_queue *rxq)
2229{
2230	unsigned i;
2231	uint16_t len;
2232
2233	if (!rxq) {
2234		PMD_DRV_LOG(DEBUG, "Pointer to rxq is NULL");
2235		return;
2236	}
2237
2238#ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC
2239	if (check_rx_burst_bulk_alloc_preconditions(rxq) == 0)
2240		len = (uint16_t)(rxq->nb_rx_desc + RTE_PMD_I40E_RX_MAX_BURST);
2241	else
2242#endif /* RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC */
2243		len = rxq->nb_rx_desc;
2244
2245	for (i = 0; i < len * sizeof(union i40e_rx_desc); i++)
2246		((volatile char *)rxq->rx_ring)[i] = 0;
2247
2248#ifdef RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC
2249	memset(&rxq->fake_mbuf, 0x0, sizeof(rxq->fake_mbuf));
2250	for (i = 0; i < RTE_PMD_I40E_RX_MAX_BURST; ++i)
2251		rxq->sw_ring[rxq->nb_rx_desc + i].mbuf = &rxq->fake_mbuf;
2252
2253	rxq->rx_nb_avail = 0;
2254	rxq->rx_next_avail = 0;
2255	rxq->rx_free_trigger = (uint16_t)(rxq->rx_free_thresh - 1);
2256#endif /* RTE_LIBRTE_I40E_RX_ALLOW_BULK_ALLOC */
2257	rxq->rx_tail = 0;
2258	rxq->nb_rx_hold = 0;
2259	rxq->pkt_first_seg = NULL;
2260	rxq->pkt_last_seg = NULL;
2261
2262	rxq->rxrearm_start = 0;
2263	rxq->rxrearm_nb = 0;
2264}
2265
2266void
2267i40e_tx_queue_release_mbufs(struct i40e_tx_queue *txq)
2268{
2269	uint16_t i;
2270
2271	if (!txq || !txq->sw_ring) {
2272		PMD_DRV_LOG(DEBUG, "Pointer to rxq or sw_ring is NULL");
2273		return;
2274	}
2275
2276	for (i = 0; i < txq->nb_tx_desc; i++) {
2277		if (txq->sw_ring[i].mbuf) {
2278			rte_pktmbuf_free_seg(txq->sw_ring[i].mbuf);
2279			txq->sw_ring[i].mbuf = NULL;
2280		}
2281	}
2282}
2283
2284void
2285i40e_reset_tx_queue(struct i40e_tx_queue *txq)
2286{
2287	struct i40e_tx_entry *txe;
2288	uint16_t i, prev, size;
2289
2290	if (!txq) {
2291		PMD_DRV_LOG(DEBUG, "Pointer to txq is NULL");
2292		return;
2293	}
2294
2295	txe = txq->sw_ring;
2296	size = sizeof(struct i40e_tx_desc) * txq->nb_tx_desc;
2297	for (i = 0; i < size; i++)
2298		((volatile char *)txq->tx_ring)[i] = 0;
2299
2300	prev = (uint16_t)(txq->nb_tx_desc - 1);
2301	for (i = 0; i < txq->nb_tx_desc; i++) {
2302		volatile struct i40e_tx_desc *txd = &txq->tx_ring[i];
2303
2304		txd->cmd_type_offset_bsz =
2305			rte_cpu_to_le_64(I40E_TX_DESC_DTYPE_DESC_DONE);
2306		txe[i].mbuf =  NULL;
2307		txe[i].last_id = i;
2308		txe[prev].next_id = i;
2309		prev = i;
2310	}
2311
2312	txq->tx_next_dd = (uint16_t)(txq->tx_rs_thresh - 1);
2313	txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
2314
2315	txq->tx_tail = 0;
2316	txq->nb_tx_used = 0;
2317
2318	txq->last_desc_cleaned = (uint16_t)(txq->nb_tx_desc - 1);
2319	txq->nb_tx_free = (uint16_t)(txq->nb_tx_desc - 1);
2320}
2321
2322/* Init the TX queue in hardware */
2323int
2324i40e_tx_queue_init(struct i40e_tx_queue *txq)
2325{
2326	enum i40e_status_code err = I40E_SUCCESS;
2327	struct i40e_vsi *vsi = txq->vsi;
2328	struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2329	uint16_t pf_q = txq->reg_idx;
2330	struct i40e_hmc_obj_txq tx_ctx;
2331	uint32_t qtx_ctl;
2332
2333	/* clear the context structure first */
2334	memset(&tx_ctx, 0, sizeof(tx_ctx));
2335	tx_ctx.new_context = 1;
2336	tx_ctx.base = txq->tx_ring_phys_addr / I40E_QUEUE_BASE_ADDR_UNIT;
2337	tx_ctx.qlen = txq->nb_tx_desc;
2338
2339#ifdef RTE_LIBRTE_IEEE1588
2340	tx_ctx.timesync_ena = 1;
2341#endif
2342	tx_ctx.rdylist = rte_le_to_cpu_16(vsi->info.qs_handle[txq->dcb_tc]);
2343	if (vsi->type == I40E_VSI_FDIR)
2344		tx_ctx.fd_ena = TRUE;
2345
2346	err = i40e_clear_lan_tx_queue_context(hw, pf_q);
2347	if (err != I40E_SUCCESS) {
2348		PMD_DRV_LOG(ERR, "Failure of clean lan tx queue context");
2349		return err;
2350	}
2351
2352	err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx);
2353	if (err != I40E_SUCCESS) {
2354		PMD_DRV_LOG(ERR, "Failure of set lan tx queue context");
2355		return err;
2356	}
2357
2358	/* Now associate this queue with this PCI function */
2359	qtx_ctl = I40E_QTX_CTL_PF_QUEUE;
2360	qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) &
2361					I40E_QTX_CTL_PF_INDX_MASK);
2362	I40E_WRITE_REG(hw, I40E_QTX_CTL(pf_q), qtx_ctl);
2363	I40E_WRITE_FLUSH(hw);
2364
2365	txq->qtx_tail = hw->hw_addr + I40E_QTX_TAIL(pf_q);
2366
2367	return err;
2368}
2369
2370int
2371i40e_alloc_rx_queue_mbufs(struct i40e_rx_queue *rxq)
2372{
2373	struct i40e_rx_entry *rxe = rxq->sw_ring;
2374	uint64_t dma_addr;
2375	uint16_t i;
2376
2377	for (i = 0; i < rxq->nb_rx_desc; i++) {
2378		volatile union i40e_rx_desc *rxd;
2379		struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mp);
2380
2381		if (unlikely(!mbuf)) {
2382			PMD_DRV_LOG(ERR, "Failed to allocate mbuf for RX");
2383			return -ENOMEM;
2384		}
2385
2386		rte_mbuf_refcnt_set(mbuf, 1);
2387		mbuf->next = NULL;
2388		mbuf->data_off = RTE_PKTMBUF_HEADROOM;
2389		mbuf->nb_segs = 1;
2390		mbuf->port = rxq->port_id;
2391
2392		dma_addr =
2393			rte_cpu_to_le_64(rte_mbuf_data_dma_addr_default(mbuf));
2394
2395		rxd = &rxq->rx_ring[i];
2396		rxd->read.pkt_addr = dma_addr;
2397		rxd->read.hdr_addr = 0;
2398#ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC
2399		rxd->read.rsvd1 = 0;
2400		rxd->read.rsvd2 = 0;
2401#endif /* RTE_LIBRTE_I40E_16BYTE_RX_DESC */
2402
2403		rxe[i].mbuf = mbuf;
2404	}
2405
2406	return 0;
2407}
2408
2409/*
2410 * Calculate the buffer length, and check the jumbo frame
2411 * and maximum packet length.
2412 */
2413static int
2414i40e_rx_queue_config(struct i40e_rx_queue *rxq)
2415{
2416	struct i40e_pf *pf = I40E_VSI_TO_PF(rxq->vsi);
2417	struct i40e_hw *hw = I40E_VSI_TO_HW(rxq->vsi);
2418	struct rte_eth_dev_data *data = pf->dev_data;
2419	uint16_t buf_size, len;
2420
2421	buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
2422		RTE_PKTMBUF_HEADROOM);
2423
2424	switch (pf->flags & (I40E_FLAG_HEADER_SPLIT_DISABLED |
2425			I40E_FLAG_HEADER_SPLIT_ENABLED)) {
2426	case I40E_FLAG_HEADER_SPLIT_ENABLED: /* Not supported */
2427		rxq->rx_hdr_len = RTE_ALIGN(I40E_RXBUF_SZ_1024,
2428				(1 << I40E_RXQ_CTX_HBUFF_SHIFT));
2429		rxq->rx_buf_len = RTE_ALIGN(I40E_RXBUF_SZ_2048,
2430				(1 << I40E_RXQ_CTX_DBUFF_SHIFT));
2431		rxq->hs_mode = i40e_header_split_enabled;
2432		break;
2433	case I40E_FLAG_HEADER_SPLIT_DISABLED:
2434	default:
2435		rxq->rx_hdr_len = 0;
2436		rxq->rx_buf_len = RTE_ALIGN(buf_size,
2437			(1 << I40E_RXQ_CTX_DBUFF_SHIFT));
2438		rxq->hs_mode = i40e_header_split_none;
2439		break;
2440	}
2441
2442	len = hw->func_caps.rx_buf_chain_len * rxq->rx_buf_len;
2443	rxq->max_pkt_len = RTE_MIN(len, data->dev_conf.rxmode.max_rx_pkt_len);
2444	if (data->dev_conf.rxmode.jumbo_frame == 1) {
2445		if (rxq->max_pkt_len <= ETHER_MAX_LEN ||
2446			rxq->max_pkt_len > I40E_FRAME_SIZE_MAX) {
2447			PMD_DRV_LOG(ERR, "maximum packet length must "
2448				    "be larger than %u and smaller than %u,"
2449				    "as jumbo frame is enabled",
2450				    (uint32_t)ETHER_MAX_LEN,
2451				    (uint32_t)I40E_FRAME_SIZE_MAX);
2452			return I40E_ERR_CONFIG;
2453		}
2454	} else {
2455		if (rxq->max_pkt_len < ETHER_MIN_LEN ||
2456			rxq->max_pkt_len > ETHER_MAX_LEN) {
2457			PMD_DRV_LOG(ERR, "maximum packet length must be "
2458				    "larger than %u and smaller than %u, "
2459				    "as jumbo frame is disabled",
2460				    (uint32_t)ETHER_MIN_LEN,
2461				    (uint32_t)ETHER_MAX_LEN);
2462			return I40E_ERR_CONFIG;
2463		}
2464	}
2465
2466	return 0;
2467}
2468
2469/* Init the RX queue in hardware */
2470int
2471i40e_rx_queue_init(struct i40e_rx_queue *rxq)
2472{
2473	int err = I40E_SUCCESS;
2474	struct i40e_hw *hw = I40E_VSI_TO_HW(rxq->vsi);
2475	struct rte_eth_dev_data *dev_data = I40E_VSI_TO_DEV_DATA(rxq->vsi);
2476	uint16_t pf_q = rxq->reg_idx;
2477	uint16_t buf_size;
2478	struct i40e_hmc_obj_rxq rx_ctx;
2479
2480	err = i40e_rx_queue_config(rxq);
2481	if (err < 0) {
2482		PMD_DRV_LOG(ERR, "Failed to config RX queue");
2483		return err;
2484	}
2485
2486	/* Clear the context structure first */
2487	memset(&rx_ctx, 0, sizeof(struct i40e_hmc_obj_rxq));
2488	rx_ctx.dbuff = rxq->rx_buf_len >> I40E_RXQ_CTX_DBUFF_SHIFT;
2489	rx_ctx.hbuff = rxq->rx_hdr_len >> I40E_RXQ_CTX_HBUFF_SHIFT;
2490
2491	rx_ctx.base = rxq->rx_ring_phys_addr / I40E_QUEUE_BASE_ADDR_UNIT;
2492	rx_ctx.qlen = rxq->nb_rx_desc;
2493#ifndef RTE_LIBRTE_I40E_16BYTE_RX_DESC
2494	rx_ctx.dsize = 1;
2495#endif
2496	rx_ctx.dtype = rxq->hs_mode;
2497	if (rxq->hs_mode)
2498		rx_ctx.hsplit_0 = I40E_HEADER_SPLIT_ALL;
2499	else
2500		rx_ctx.hsplit_0 = I40E_HEADER_SPLIT_NONE;
2501	rx_ctx.rxmax = rxq->max_pkt_len;
2502	rx_ctx.tphrdesc_ena = 1;
2503	rx_ctx.tphwdesc_ena = 1;
2504	rx_ctx.tphdata_ena = 1;
2505	rx_ctx.tphhead_ena = 1;
2506	rx_ctx.lrxqthresh = 2;
2507	rx_ctx.crcstrip = (rxq->crc_len == 0) ? 1 : 0;
2508	rx_ctx.l2tsel = 1;
2509	/* showiv indicates if inner VLAN is stripped inside of tunnel
2510	 * packet. When set it to 1, vlan information is stripped from
2511	 * the inner header, but the hardware does not put it in the
2512	 * descriptor. So set it zero by default.
2513	 */
2514	rx_ctx.showiv = 0;
2515	rx_ctx.prefena = 1;
2516
2517	err = i40e_clear_lan_rx_queue_context(hw, pf_q);
2518	if (err != I40E_SUCCESS) {
2519		PMD_DRV_LOG(ERR, "Failed to clear LAN RX queue context");
2520		return err;
2521	}
2522	err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx);
2523	if (err != I40E_SUCCESS) {
2524		PMD_DRV_LOG(ERR, "Failed to set LAN RX queue context");
2525		return err;
2526	}
2527
2528	rxq->qrx_tail = hw->hw_addr + I40E_QRX_TAIL(pf_q);
2529
2530	buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mp) -
2531		RTE_PKTMBUF_HEADROOM);
2532
2533	/* Check if scattered RX needs to be used. */
2534	if ((rxq->max_pkt_len + 2 * I40E_VLAN_TAG_SIZE) > buf_size) {
2535		dev_data->scattered_rx = 1;
2536	}
2537
2538	/* Init the RX tail regieter. */
2539	I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
2540
2541	return 0;
2542}
2543
2544void
2545i40e_dev_clear_queues(struct rte_eth_dev *dev)
2546{
2547	uint16_t i;
2548
2549	PMD_INIT_FUNC_TRACE();
2550
2551	for (i = 0; i < dev->data->nb_tx_queues; i++) {
2552		if (!dev->data->tx_queues[i])
2553			continue;
2554		i40e_tx_queue_release_mbufs(dev->data->tx_queues[i]);
2555		i40e_reset_tx_queue(dev->data->tx_queues[i]);
2556	}
2557
2558	for (i = 0; i < dev->data->nb_rx_queues; i++) {
2559		if (!dev->data->rx_queues[i])
2560			continue;
2561		i40e_rx_queue_release_mbufs(dev->data->rx_queues[i]);
2562		i40e_reset_rx_queue(dev->data->rx_queues[i]);
2563	}
2564}
2565
2566void
2567i40e_dev_free_queues(struct rte_eth_dev *dev)
2568{
2569	uint16_t i;
2570
2571	PMD_INIT_FUNC_TRACE();
2572
2573	for (i = 0; i < dev->data->nb_rx_queues; i++) {
2574		if (!dev->data->rx_queues[i])
2575			continue;
2576		i40e_dev_rx_queue_release(dev->data->rx_queues[i]);
2577		dev->data->rx_queues[i] = NULL;
2578	}
2579	dev->data->nb_rx_queues = 0;
2580
2581	for (i = 0; i < dev->data->nb_tx_queues; i++) {
2582		if (!dev->data->tx_queues[i])
2583			continue;
2584		i40e_dev_tx_queue_release(dev->data->tx_queues[i]);
2585		dev->data->tx_queues[i] = NULL;
2586	}
2587	dev->data->nb_tx_queues = 0;
2588}
2589
2590#define I40E_FDIR_NUM_TX_DESC  I40E_MIN_RING_DESC
2591#define I40E_FDIR_NUM_RX_DESC  I40E_MIN_RING_DESC
2592
2593enum i40e_status_code
2594i40e_fdir_setup_tx_resources(struct i40e_pf *pf)
2595{
2596	struct i40e_tx_queue *txq;
2597	const struct rte_memzone *tz = NULL;
2598	uint32_t ring_size;
2599	struct rte_eth_dev *dev;
2600
2601	if (!pf) {
2602		PMD_DRV_LOG(ERR, "PF is not available");
2603		return I40E_ERR_BAD_PTR;
2604	}
2605
2606	dev = pf->adapter->eth_dev;
2607
2608	/* Allocate the TX queue data structure. */
2609	txq = rte_zmalloc_socket("i40e fdir tx queue",
2610				  sizeof(struct i40e_tx_queue),
2611				  RTE_CACHE_LINE_SIZE,
2612				  SOCKET_ID_ANY);
2613	if (!txq) {
2614		PMD_DRV_LOG(ERR, "Failed to allocate memory for "
2615					"tx queue structure.");
2616		return I40E_ERR_NO_MEMORY;
2617	}
2618
2619	/* Allocate TX hardware ring descriptors. */
2620	ring_size = sizeof(struct i40e_tx_desc) * I40E_FDIR_NUM_TX_DESC;
2621	ring_size = RTE_ALIGN(ring_size, I40E_DMA_MEM_ALIGN);
2622
2623	tz = rte_eth_dma_zone_reserve(dev, "fdir_tx_ring",
2624				      I40E_FDIR_QUEUE_ID, ring_size,
2625				      I40E_RING_BASE_ALIGN, SOCKET_ID_ANY);
2626	if (!tz) {
2627		i40e_dev_tx_queue_release(txq);
2628		PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for TX.");
2629		return I40E_ERR_NO_MEMORY;
2630	}
2631
2632	txq->nb_tx_desc = I40E_FDIR_NUM_TX_DESC;
2633	txq->queue_id = I40E_FDIR_QUEUE_ID;
2634	txq->reg_idx = pf->fdir.fdir_vsi->base_queue;
2635	txq->vsi = pf->fdir.fdir_vsi;
2636
2637	txq->tx_ring_phys_addr = rte_mem_phy2mch(tz->memseg_id, tz->phys_addr);
2638	txq->tx_ring = (struct i40e_tx_desc *)tz->addr;
2639	/*
2640	 * don't need to allocate software ring and reset for the fdir
2641	 * program queue just set the queue has been configured.
2642	 */
2643	txq->q_set = TRUE;
2644	pf->fdir.txq = txq;
2645
2646	return I40E_SUCCESS;
2647}
2648
2649enum i40e_status_code
2650i40e_fdir_setup_rx_resources(struct i40e_pf *pf)
2651{
2652	struct i40e_rx_queue *rxq;
2653	const struct rte_memzone *rz = NULL;
2654	uint32_t ring_size;
2655	struct rte_eth_dev *dev;
2656
2657	if (!pf) {
2658		PMD_DRV_LOG(ERR, "PF is not available");
2659		return I40E_ERR_BAD_PTR;
2660	}
2661
2662	dev = pf->adapter->eth_dev;
2663
2664	/* Allocate the RX queue data structure. */
2665	rxq = rte_zmalloc_socket("i40e fdir rx queue",
2666				  sizeof(struct i40e_rx_queue),
2667				  RTE_CACHE_LINE_SIZE,
2668				  SOCKET_ID_ANY);
2669	if (!rxq) {
2670		PMD_DRV_LOG(ERR, "Failed to allocate memory for "
2671					"rx queue structure.");
2672		return I40E_ERR_NO_MEMORY;
2673	}
2674
2675	/* Allocate RX hardware ring descriptors. */
2676	ring_size = sizeof(union i40e_rx_desc) * I40E_FDIR_NUM_RX_DESC;
2677	ring_size = RTE_ALIGN(ring_size, I40E_DMA_MEM_ALIGN);
2678
2679	rz = rte_eth_dma_zone_reserve(dev, "fdir_rx_ring",
2680				      I40E_FDIR_QUEUE_ID, ring_size,
2681				      I40E_RING_BASE_ALIGN, SOCKET_ID_ANY);
2682	if (!rz) {
2683		i40e_dev_rx_queue_release(rxq);
2684		PMD_DRV_LOG(ERR, "Failed to reserve DMA memory for RX.");
2685		return I40E_ERR_NO_MEMORY;
2686	}
2687
2688	rxq->nb_rx_desc = I40E_FDIR_NUM_RX_DESC;
2689	rxq->queue_id = I40E_FDIR_QUEUE_ID;
2690	rxq->reg_idx = pf->fdir.fdir_vsi->base_queue;
2691	rxq->vsi = pf->fdir.fdir_vsi;
2692
2693	rxq->rx_ring_phys_addr = rte_mem_phy2mch(rz->memseg_id, rz->phys_addr);
2694	rxq->rx_ring = (union i40e_rx_desc *)rz->addr;
2695
2696	/*
2697	 * Don't need to allocate software ring and reset for the fdir
2698	 * rx queue, just set the queue has been configured.
2699	 */
2700	rxq->q_set = TRUE;
2701	pf->fdir.rxq = rxq;
2702
2703	return I40E_SUCCESS;
2704}
2705
2706void
2707i40e_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
2708	struct rte_eth_rxq_info *qinfo)
2709{
2710	struct i40e_rx_queue *rxq;
2711
2712	rxq = dev->data->rx_queues[queue_id];
2713
2714	qinfo->mp = rxq->mp;
2715	qinfo->scattered_rx = dev->data->scattered_rx;
2716	qinfo->nb_desc = rxq->nb_rx_desc;
2717
2718	qinfo->conf.rx_free_thresh = rxq->rx_free_thresh;
2719	qinfo->conf.rx_drop_en = rxq->drop_en;
2720	qinfo->conf.rx_deferred_start = rxq->rx_deferred_start;
2721}
2722
2723void
2724i40e_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
2725	struct rte_eth_txq_info *qinfo)
2726{
2727	struct i40e_tx_queue *txq;
2728
2729	txq = dev->data->tx_queues[queue_id];
2730
2731	qinfo->nb_desc = txq->nb_tx_desc;
2732
2733	qinfo->conf.tx_thresh.pthresh = txq->pthresh;
2734	qinfo->conf.tx_thresh.hthresh = txq->hthresh;
2735	qinfo->conf.tx_thresh.wthresh = txq->wthresh;
2736
2737	qinfo->conf.tx_free_thresh = txq->tx_free_thresh;
2738	qinfo->conf.tx_rs_thresh = txq->tx_rs_thresh;
2739	qinfo->conf.txq_flags = txq->txq_flags;
2740	qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
2741}
2742
2743void __attribute__((cold))
2744i40e_set_rx_function(struct rte_eth_dev *dev)
2745{
2746	struct i40e_adapter *ad =
2747		I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
2748	uint16_t rx_using_sse, i;
2749	/* In order to allow Vector Rx there are a few configuration
2750	 * conditions to be met and Rx Bulk Allocation should be allowed.
2751	 */
2752	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
2753		if (i40e_rx_vec_dev_conf_condition_check(dev) ||
2754		    !ad->rx_bulk_alloc_allowed) {
2755			PMD_INIT_LOG(DEBUG, "Port[%d] doesn't meet"
2756				     " Vector Rx preconditions",
2757				     dev->data->port_id);
2758
2759			ad->rx_vec_allowed = false;
2760		}
2761		if (ad->rx_vec_allowed) {
2762			for (i = 0; i < dev->data->nb_rx_queues; i++) {
2763				struct i40e_rx_queue *rxq =
2764					dev->data->rx_queues[i];
2765
2766				if (rxq && i40e_rxq_vec_setup(rxq)) {
2767					ad->rx_vec_allowed = false;
2768					break;
2769				}
2770			}
2771		}
2772	}
2773
2774	if (dev->data->scattered_rx) {
2775		/* Set the non-LRO scattered callback: there are Vector and
2776		 * single allocation versions.
2777		 */
2778		if (ad->rx_vec_allowed) {
2779			PMD_INIT_LOG(DEBUG, "Using Vector Scattered Rx "
2780					    "callback (port=%d).",
2781				     dev->data->port_id);
2782
2783			dev->rx_pkt_burst = i40e_recv_scattered_pkts_vec;
2784		} else {
2785			PMD_INIT_LOG(DEBUG, "Using a Scattered with bulk "
2786					   "allocation callback (port=%d).",
2787				     dev->data->port_id);
2788			dev->rx_pkt_burst = i40e_recv_scattered_pkts;
2789		}
2790	/* If parameters allow we are going to choose between the following
2791	 * callbacks:
2792	 *    - Vector
2793	 *    - Bulk Allocation
2794	 *    - Single buffer allocation (the simplest one)
2795	 */
2796	} else if (ad->rx_vec_allowed) {
2797		PMD_INIT_LOG(DEBUG, "Vector rx enabled, please make sure RX "
2798				    "burst size no less than %d (port=%d).",
2799			     RTE_I40E_DESCS_PER_LOOP,
2800			     dev->data->port_id);
2801
2802		dev->rx_pkt_burst = i40e_recv_pkts_vec;
2803	} else if (ad->rx_bulk_alloc_allowed) {
2804		PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are "
2805				    "satisfied. Rx Burst Bulk Alloc function "
2806				    "will be used on port=%d.",
2807			     dev->data->port_id);
2808
2809		dev->rx_pkt_burst = i40e_recv_pkts_bulk_alloc;
2810	} else {
2811		PMD_INIT_LOG(DEBUG, "Rx Burst Bulk Alloc Preconditions are not "
2812				    "satisfied, or Scattered Rx is requested "
2813				    "(port=%d).",
2814			     dev->data->port_id);
2815
2816		dev->rx_pkt_burst = i40e_recv_pkts;
2817	}
2818
2819	/* Propagate information about RX function choice through all queues. */
2820	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
2821		rx_using_sse =
2822			(dev->rx_pkt_burst == i40e_recv_scattered_pkts_vec ||
2823			 dev->rx_pkt_burst == i40e_recv_pkts_vec);
2824
2825		for (i = 0; i < dev->data->nb_rx_queues; i++) {
2826			struct i40e_rx_queue *rxq = dev->data->rx_queues[i];
2827
2828			if (rxq)
2829				rxq->rx_using_sse = rx_using_sse;
2830		}
2831	}
2832}
2833
2834void __attribute__((cold))
2835i40e_set_tx_function_flag(struct rte_eth_dev *dev, struct i40e_tx_queue *txq)
2836{
2837	struct i40e_adapter *ad =
2838		I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
2839
2840	/* Use a simple Tx queue (no offloads, no multi segs) if possible */
2841	if (((txq->txq_flags & I40E_SIMPLE_FLAGS) == I40E_SIMPLE_FLAGS)
2842			&& (txq->tx_rs_thresh >= RTE_PMD_I40E_TX_MAX_BURST)) {
2843		if (txq->tx_rs_thresh <= RTE_I40E_TX_MAX_FREE_BUF_SZ) {
2844			PMD_INIT_LOG(DEBUG, "Vector tx"
2845				     " can be enabled on this txq.");
2846
2847		} else {
2848			ad->tx_vec_allowed = false;
2849		}
2850	} else {
2851		ad->tx_simple_allowed = false;
2852	}
2853}
2854
2855void __attribute__((cold))
2856i40e_set_tx_function(struct rte_eth_dev *dev)
2857{
2858	struct i40e_adapter *ad =
2859		I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
2860	int i;
2861
2862	if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
2863		if (ad->tx_vec_allowed) {
2864			for (i = 0; i < dev->data->nb_tx_queues; i++) {
2865				struct i40e_tx_queue *txq =
2866					dev->data->tx_queues[i];
2867
2868				if (txq && i40e_txq_vec_setup(txq)) {
2869					ad->tx_vec_allowed = false;
2870					break;
2871				}
2872			}
2873		}
2874	}
2875
2876	if (ad->tx_simple_allowed) {
2877		if (ad->tx_vec_allowed) {
2878			PMD_INIT_LOG(DEBUG, "Vector tx finally be used.");
2879			dev->tx_pkt_burst = i40e_xmit_pkts_vec;
2880		} else {
2881			PMD_INIT_LOG(DEBUG, "Simple tx finally be used.");
2882			dev->tx_pkt_burst = i40e_xmit_pkts_simple;
2883		}
2884		dev->tx_pkt_prepare = NULL;
2885	} else {
2886		PMD_INIT_LOG(DEBUG, "Xmit tx finally be used.");
2887		dev->tx_pkt_burst = i40e_xmit_pkts;
2888		dev->tx_pkt_prepare = i40e_prep_pkts;
2889	}
2890}
2891
2892/* Stubs needed for linkage when CONFIG_RTE_I40E_INC_VECTOR is set to 'n' */
2893int __attribute__((weak))
2894i40e_rx_vec_dev_conf_condition_check(struct rte_eth_dev __rte_unused *dev)
2895{
2896	return -1;
2897}
2898
2899uint16_t __attribute__((weak))
2900i40e_recv_pkts_vec(
2901	void __rte_unused *rx_queue,
2902	struct rte_mbuf __rte_unused **rx_pkts,
2903	uint16_t __rte_unused nb_pkts)
2904{
2905	return 0;
2906}
2907
2908uint16_t __attribute__((weak))
2909i40e_recv_scattered_pkts_vec(
2910	void __rte_unused *rx_queue,
2911	struct rte_mbuf __rte_unused **rx_pkts,
2912	uint16_t __rte_unused nb_pkts)
2913{
2914	return 0;
2915}
2916
2917int __attribute__((weak))
2918i40e_rxq_vec_setup(struct i40e_rx_queue __rte_unused *rxq)
2919{
2920	return -1;
2921}
2922
2923int __attribute__((weak))
2924i40e_txq_vec_setup(struct i40e_tx_queue __rte_unused *txq)
2925{
2926	return -1;
2927}
2928
2929void __attribute__((weak))
2930i40e_rx_queue_release_mbufs_vec(struct i40e_rx_queue __rte_unused*rxq)
2931{
2932	return;
2933}
2934
2935uint16_t __attribute__((weak))
2936i40e_xmit_pkts_vec(void __rte_unused *tx_queue,
2937		   struct rte_mbuf __rte_unused **tx_pkts,
2938		   uint16_t __rte_unused nb_pkts)
2939{
2940	return 0;
2941}
2942