1/*-
2 *   BSD LICENSE
3 *
4 *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
5 *   All rights reserved.
6 *
7 *   Redistribution and use in source and binary forms, with or without
8 *   modification, are permitted provided that the following conditions
9 *   are met:
10 *
11 *     * Redistributions of source code must retain the above copyright
12 *       notice, this list of conditions and the following disclaimer.
13 *     * Redistributions in binary form must reproduce the above copyright
14 *       notice, this list of conditions and the following disclaimer in
15 *       the documentation and/or other materials provided with the
16 *       distribution.
17 *     * Neither the name of Intel Corporation nor the names of its
18 *       contributors may be used to endorse or promote products derived
19 *       from this software without specific prior written permission.
20 *
21 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34#include <stdint.h>
35#include <rte_ethdev.h>
36#include <rte_malloc.h>
37
38#include "ixgbe_ethdev.h"
39#include "ixgbe_rxtx.h"
40#include "ixgbe_rxtx_vec_common.h"
41
42#include <arm_neon.h>
43
44#pragma GCC diagnostic ignored "-Wcast-qual"
45
46static inline void
47ixgbe_rxq_rearm(struct ixgbe_rx_queue *rxq)
48{
49	int i;
50	uint16_t rx_id;
51	volatile union ixgbe_adv_rx_desc *rxdp;
52	struct ixgbe_rx_entry *rxep = &rxq->sw_ring[rxq->rxrearm_start];
53	struct rte_mbuf *mb0, *mb1;
54	uint64x2_t dma_addr0, dma_addr1;
55	uint64x2_t zero = vdupq_n_u64(0);
56	uint64_t paddr;
57	uint8x8_t p;
58
59	rxdp = rxq->rx_ring + rxq->rxrearm_start;
60
61	/* Pull 'n' more MBUFs into the software ring */
62	if (unlikely(rte_mempool_get_bulk(rxq->mb_pool,
63					  (void *)rxep,
64					  RTE_IXGBE_RXQ_REARM_THRESH) < 0)) {
65		if (rxq->rxrearm_nb + RTE_IXGBE_RXQ_REARM_THRESH >=
66		    rxq->nb_rx_desc) {
67			for (i = 0; i < RTE_IXGBE_DESCS_PER_LOOP; i++) {
68				rxep[i].mbuf = &rxq->fake_mbuf;
69				vst1q_u64((uint64_t *)&rxdp[i].read,
70					  zero);
71			}
72		}
73		rte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed +=
74			RTE_IXGBE_RXQ_REARM_THRESH;
75		return;
76	}
77
78	p = vld1_u8((uint8_t *)&rxq->mbuf_initializer);
79
80	/* Initialize the mbufs in vector, process 2 mbufs in one loop */
81	for (i = 0; i < RTE_IXGBE_RXQ_REARM_THRESH; i += 2, rxep += 2) {
82		mb0 = rxep[0].mbuf;
83		mb1 = rxep[1].mbuf;
84
85		/*
86		 * Flush mbuf with pkt template.
87		 * Data to be rearmed is 6 bytes long.
88		 * Though, RX will overwrite ol_flags that are coming next
89		 * anyway. So overwrite whole 8 bytes with one load:
90		 * 6 bytes of rearm_data plus first 2 bytes of ol_flags.
91		 */
92		vst1_u8((uint8_t *)&mb0->rearm_data, p);
93		paddr = mb0->buf_physaddr + RTE_PKTMBUF_HEADROOM;
94		dma_addr0 = vsetq_lane_u64(paddr, zero, 0);
95		/* flush desc with pa dma_addr */
96		vst1q_u64((uint64_t *)&rxdp++->read, dma_addr0);
97
98		vst1_u8((uint8_t *)&mb1->rearm_data, p);
99		paddr = mb1->buf_physaddr + RTE_PKTMBUF_HEADROOM;
100		dma_addr1 = vsetq_lane_u64(paddr, zero, 0);
101		vst1q_u64((uint64_t *)&rxdp++->read, dma_addr1);
102	}
103
104	rxq->rxrearm_start += RTE_IXGBE_RXQ_REARM_THRESH;
105	if (rxq->rxrearm_start >= rxq->nb_rx_desc)
106		rxq->rxrearm_start = 0;
107
108	rxq->rxrearm_nb -= RTE_IXGBE_RXQ_REARM_THRESH;
109
110	rx_id = (uint16_t)((rxq->rxrearm_start == 0) ?
111			     (rxq->nb_rx_desc - 1) : (rxq->rxrearm_start - 1));
112
113	/* Update the tail pointer on the NIC */
114	IXGBE_PCI_REG_WRITE(rxq->rdt_reg_addr, rx_id);
115}
116
117/* Handling the offload flags (olflags) field takes computation
118 * time when receiving packets. Therefore we provide a flag to disable
119 * the processing of the olflags field when they are not needed. This
120 * gives improved performance, at the cost of losing the offload info
121 * in the received packet
122 */
123#ifdef RTE_IXGBE_RX_OLFLAGS_ENABLE
124
125#define VTAG_SHIFT     (3)
126
127static inline void
128desc_to_olflags_v(uint8x16x2_t sterr_tmp1, uint8x16x2_t sterr_tmp2,
129		  uint8x16_t staterr, struct rte_mbuf **rx_pkts)
130{
131	uint8x16_t ptype;
132	uint8x16_t vtag;
133
134	union {
135		uint8_t e[4];
136		uint32_t word;
137	} vol;
138
139	const uint8x16_t pkttype_msk = {
140			PKT_RX_VLAN_PKT, PKT_RX_VLAN_PKT,
141			PKT_RX_VLAN_PKT, PKT_RX_VLAN_PKT,
142			0x00, 0x00, 0x00, 0x00,
143			0x00, 0x00, 0x00, 0x00,
144			0x00, 0x00, 0x00, 0x00};
145
146	const uint8x16_t rsstype_msk = {
147			0x0F, 0x0F, 0x0F, 0x0F,
148			0x00, 0x00, 0x00, 0x00,
149			0x00, 0x00, 0x00, 0x00,
150			0x00, 0x00, 0x00, 0x00};
151
152	const uint8x16_t rss_flags = {
153			0, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH, PKT_RX_RSS_HASH,
154			0, PKT_RX_RSS_HASH, 0, PKT_RX_RSS_HASH,
155			PKT_RX_RSS_HASH, 0, 0, 0,
156			0, 0, 0, PKT_RX_FDIR};
157
158	ptype = vzipq_u8(sterr_tmp1.val[0], sterr_tmp2.val[0]).val[0];
159	ptype = vandq_u8(ptype, rsstype_msk);
160	ptype = vqtbl1q_u8(rss_flags, ptype);
161
162	vtag = vshrq_n_u8(staterr, VTAG_SHIFT);
163	vtag = vandq_u8(vtag, pkttype_msk);
164	vtag = vorrq_u8(ptype, vtag);
165
166	vol.word = vgetq_lane_u32(vreinterpretq_u32_u8(vtag), 0);
167
168	rx_pkts[0]->ol_flags = vol.e[0];
169	rx_pkts[1]->ol_flags = vol.e[1];
170	rx_pkts[2]->ol_flags = vol.e[2];
171	rx_pkts[3]->ol_flags = vol.e[3];
172}
173#else
174#define desc_to_olflags_v(sterr_tmp1, sterr_tmp2, staterr, rx_pkts)
175#endif
176
177/*
178 * vPMD raw receive routine, only accept(nb_pkts >= RTE_IXGBE_DESCS_PER_LOOP)
179 *
180 * Notice:
181 * - nb_pkts < RTE_IXGBE_DESCS_PER_LOOP, just return no packet
182 * - nb_pkts > RTE_IXGBE_MAX_RX_BURST, only scan RTE_IXGBE_MAX_RX_BURST
183 *   numbers of DD bit
184 * - floor align nb_pkts to a RTE_IXGBE_DESC_PER_LOOP power-of-two
185 * - don't support ol_flags for rss and csum err
186 */
187
188#define IXGBE_VPMD_DESC_DD_MASK		0x01010101
189#define IXGBE_VPMD_DESC_EOP_MASK	0x02020202
190
191static inline uint16_t
192_recv_raw_pkts_vec(struct ixgbe_rx_queue *rxq, struct rte_mbuf **rx_pkts,
193		   uint16_t nb_pkts, uint8_t *split_packet)
194{
195	volatile union ixgbe_adv_rx_desc *rxdp;
196	struct ixgbe_rx_entry *sw_ring;
197	uint16_t nb_pkts_recd;
198	int pos;
199	uint8x16_t shuf_msk = {
200		0xFF, 0xFF,
201		0xFF, 0xFF,  /* skip 32 bits pkt_type */
202		12, 13,      /* octet 12~13, low 16 bits pkt_len */
203		0xFF, 0xFF,  /* skip high 16 bits pkt_len, zero out */
204		12, 13,      /* octet 12~13, 16 bits data_len */
205		14, 15,      /* octet 14~15, low 16 bits vlan_macip */
206		4, 5, 6, 7  /* octet 4~7, 32bits rss */
207		};
208	uint16x8_t crc_adjust = {0, 0, rxq->crc_len, 0,
209				 rxq->crc_len, 0, 0, 0};
210
211	/* nb_pkts shall be less equal than RTE_IXGBE_MAX_RX_BURST */
212	nb_pkts = RTE_MIN(nb_pkts, RTE_IXGBE_MAX_RX_BURST);
213
214	/* nb_pkts has to be floor-aligned to RTE_IXGBE_DESCS_PER_LOOP */
215	nb_pkts = RTE_ALIGN_FLOOR(nb_pkts, RTE_IXGBE_DESCS_PER_LOOP);
216
217	/* Just the act of getting into the function from the application is
218	 * going to cost about 7 cycles
219	 */
220	rxdp = rxq->rx_ring + rxq->rx_tail;
221
222	rte_prefetch_non_temporal(rxdp);
223
224	/* See if we need to rearm the RX queue - gives the prefetch a bit
225	 * of time to act
226	 */
227	if (rxq->rxrearm_nb > RTE_IXGBE_RXQ_REARM_THRESH)
228		ixgbe_rxq_rearm(rxq);
229
230	/* Before we start moving massive data around, check to see if
231	 * there is actually a packet available
232	 */
233	if (!(rxdp->wb.upper.status_error &
234				rte_cpu_to_le_32(IXGBE_RXDADV_STAT_DD)))
235		return 0;
236
237	/* Cache is empty -> need to scan the buffer rings, but first move
238	 * the next 'n' mbufs into the cache
239	 */
240	sw_ring = &rxq->sw_ring[rxq->rx_tail];
241
242	/* A. load 4 packet in one loop
243	 * B. copy 4 mbuf point from swring to rx_pkts
244	 * C. calc the number of DD bits among the 4 packets
245	 * [C*. extract the end-of-packet bit, if requested]
246	 * D. fill info. from desc to mbuf
247	 */
248	for (pos = 0, nb_pkts_recd = 0; pos < nb_pkts;
249			pos += RTE_IXGBE_DESCS_PER_LOOP,
250			rxdp += RTE_IXGBE_DESCS_PER_LOOP) {
251		uint64x2_t descs[RTE_IXGBE_DESCS_PER_LOOP];
252		uint8x16_t pkt_mb1, pkt_mb2, pkt_mb3, pkt_mb4;
253		uint8x16x2_t sterr_tmp1, sterr_tmp2;
254		uint64x2_t mbp1, mbp2;
255		uint8x16_t staterr;
256		uint16x8_t tmp;
257		uint32_t var = 0;
258		uint32_t stat;
259
260		/* B.1 load 1 mbuf point */
261		mbp1 = vld1q_u64((uint64_t *)&sw_ring[pos]);
262
263		/* B.2 copy 2 mbuf point into rx_pkts  */
264		vst1q_u64((uint64_t *)&rx_pkts[pos], mbp1);
265
266		/* B.1 load 1 mbuf point */
267		mbp2 = vld1q_u64((uint64_t *)&sw_ring[pos + 2]);
268
269		/* A. load 4 pkts descs */
270		descs[0] =  vld1q_u64((uint64_t *)(rxdp));
271		descs[1] =  vld1q_u64((uint64_t *)(rxdp + 1));
272		descs[2] =  vld1q_u64((uint64_t *)(rxdp + 2));
273		descs[3] =  vld1q_u64((uint64_t *)(rxdp + 3));
274		rte_smp_rmb();
275
276		/* B.2 copy 2 mbuf point into rx_pkts  */
277		vst1q_u64((uint64_t *)&rx_pkts[pos + 2], mbp2);
278
279		if (split_packet) {
280			rte_mbuf_prefetch_part2(rx_pkts[pos]);
281			rte_mbuf_prefetch_part2(rx_pkts[pos + 1]);
282			rte_mbuf_prefetch_part2(rx_pkts[pos + 2]);
283			rte_mbuf_prefetch_part2(rx_pkts[pos + 3]);
284		}
285
286		/* D.1 pkt 3,4 convert format from desc to pktmbuf */
287		pkt_mb4 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[3]), shuf_msk);
288		pkt_mb3 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[2]), shuf_msk);
289
290		/* D.1 pkt 1,2 convert format from desc to pktmbuf */
291		pkt_mb2 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[1]), shuf_msk);
292		pkt_mb1 = vqtbl1q_u8(vreinterpretq_u8_u64(descs[0]), shuf_msk);
293
294		/* C.1 4=>2 filter staterr info only */
295		sterr_tmp2 = vzipq_u8(vreinterpretq_u8_u64(descs[1]),
296				      vreinterpretq_u8_u64(descs[3]));
297		/* C.1 4=>2 filter staterr info only */
298		sterr_tmp1 = vzipq_u8(vreinterpretq_u8_u64(descs[0]),
299				      vreinterpretq_u8_u64(descs[2]));
300
301		/* C.2 get 4 pkts staterr value  */
302		staterr = vzipq_u8(sterr_tmp1.val[1], sterr_tmp2.val[1]).val[0];
303		stat = vgetq_lane_u32(vreinterpretq_u32_u8(staterr), 0);
304
305		/* set ol_flags with vlan packet type */
306		desc_to_olflags_v(sterr_tmp1, sterr_tmp2, staterr,
307				  &rx_pkts[pos]);
308
309		/* D.2 pkt 3,4 set in_port/nb_seg and remove crc */
310		tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb4), crc_adjust);
311		pkt_mb4 = vreinterpretq_u8_u16(tmp);
312		tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb3), crc_adjust);
313		pkt_mb3 = vreinterpretq_u8_u16(tmp);
314
315		/* D.3 copy final 3,4 data to rx_pkts */
316		vst1q_u8((void *)&rx_pkts[pos + 3]->rx_descriptor_fields1,
317			 pkt_mb4);
318		vst1q_u8((void *)&rx_pkts[pos + 2]->rx_descriptor_fields1,
319			 pkt_mb3);
320
321		/* D.2 pkt 1,2 set in_port/nb_seg and remove crc */
322		tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb2), crc_adjust);
323		pkt_mb2 = vreinterpretq_u8_u16(tmp);
324		tmp = vsubq_u16(vreinterpretq_u16_u8(pkt_mb1), crc_adjust);
325		pkt_mb1 = vreinterpretq_u8_u16(tmp);
326
327		/* C* extract and record EOP bit */
328		if (split_packet) {
329			/* and with mask to extract bits, flipping 1-0 */
330			*(int *)split_packet = ~stat & IXGBE_VPMD_DESC_EOP_MASK;
331
332			split_packet += RTE_IXGBE_DESCS_PER_LOOP;
333
334			/* zero-out next pointers */
335			rx_pkts[pos]->next = NULL;
336			rx_pkts[pos + 1]->next = NULL;
337			rx_pkts[pos + 2]->next = NULL;
338			rx_pkts[pos + 3]->next = NULL;
339		}
340
341		rte_prefetch_non_temporal(rxdp + RTE_IXGBE_DESCS_PER_LOOP);
342
343		/* D.3 copy final 1,2 data to rx_pkts */
344		vst1q_u8((uint8_t *)&rx_pkts[pos + 1]->rx_descriptor_fields1,
345			 pkt_mb2);
346		vst1q_u8((uint8_t *)&rx_pkts[pos]->rx_descriptor_fields1,
347			 pkt_mb1);
348
349		stat &= IXGBE_VPMD_DESC_DD_MASK;
350
351		/* C.4 calc avaialbe number of desc */
352		if (likely(stat != IXGBE_VPMD_DESC_DD_MASK)) {
353			while (stat & 0x01) {
354				++var;
355				stat = stat >> 8;
356			}
357			nb_pkts_recd += var;
358			break;
359		} else {
360			nb_pkts_recd += RTE_IXGBE_DESCS_PER_LOOP;
361		}
362	}
363
364	/* Update our internal tail pointer */
365	rxq->rx_tail = (uint16_t)(rxq->rx_tail + nb_pkts_recd);
366	rxq->rx_tail = (uint16_t)(rxq->rx_tail & (rxq->nb_rx_desc - 1));
367	rxq->rxrearm_nb = (uint16_t)(rxq->rxrearm_nb + nb_pkts_recd);
368
369	return nb_pkts_recd;
370}
371
372/*
373 * vPMD receive routine, only accept(nb_pkts >= RTE_IXGBE_DESCS_PER_LOOP)
374 *
375 * Notice:
376 * - nb_pkts < RTE_IXGBE_DESCS_PER_LOOP, just return no packet
377 * - nb_pkts > RTE_IXGBE_MAX_RX_BURST, only scan RTE_IXGBE_MAX_RX_BURST
378 *   numbers of DD bit
379 * - floor align nb_pkts to a RTE_IXGBE_DESC_PER_LOOP power-of-two
380 * - don't support ol_flags for rss and csum err
381 */
382uint16_t
383ixgbe_recv_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
384		uint16_t nb_pkts)
385{
386	return _recv_raw_pkts_vec(rx_queue, rx_pkts, nb_pkts, NULL);
387}
388
389/*
390 * vPMD receive routine that reassembles scattered packets
391 *
392 * Notice:
393 * - don't support ol_flags for rss and csum err
394 * - nb_pkts < RTE_IXGBE_DESCS_PER_LOOP, just return no packet
395 * - nb_pkts > RTE_IXGBE_MAX_RX_BURST, only scan RTE_IXGBE_MAX_RX_BURST
396 *   numbers of DD bit
397 * - floor align nb_pkts to a RTE_IXGBE_DESC_PER_LOOP power-of-two
398 */
399uint16_t
400ixgbe_recv_scattered_pkts_vec(void *rx_queue, struct rte_mbuf **rx_pkts,
401		uint16_t nb_pkts)
402{
403	struct ixgbe_rx_queue *rxq = rx_queue;
404	uint8_t split_flags[RTE_IXGBE_MAX_RX_BURST] = {0};
405
406	/* get some new buffers */
407	uint16_t nb_bufs = _recv_raw_pkts_vec(rxq, rx_pkts, nb_pkts,
408			split_flags);
409	if (nb_bufs == 0)
410		return 0;
411
412	/* happy day case, full burst + no packets to be joined */
413	const uint64_t *split_fl64 = (uint64_t *)split_flags;
414	if (rxq->pkt_first_seg == NULL &&
415			split_fl64[0] == 0 && split_fl64[1] == 0 &&
416			split_fl64[2] == 0 && split_fl64[3] == 0)
417		return nb_bufs;
418
419	/* reassemble any packets that need reassembly*/
420	unsigned int i = 0;
421	if (rxq->pkt_first_seg == NULL) {
422		/* find the first split flag, and only reassemble then*/
423		while (i < nb_bufs && !split_flags[i])
424			i++;
425		if (i == nb_bufs)
426			return nb_bufs;
427	}
428	return i + reassemble_packets(rxq, &rx_pkts[i], nb_bufs - i,
429		&split_flags[i]);
430}
431
432static inline void
433vtx1(volatile union ixgbe_adv_tx_desc *txdp,
434		struct rte_mbuf *pkt, uint64_t flags)
435{
436	uint64x2_t descriptor = {
437			pkt->buf_physaddr + pkt->data_off,
438			(uint64_t)pkt->pkt_len << 46 | flags | pkt->data_len};
439
440	vst1q_u64((uint64_t *)&txdp->read, descriptor);
441}
442
443static inline void
444vtx(volatile union ixgbe_adv_tx_desc *txdp,
445		struct rte_mbuf **pkt, uint16_t nb_pkts,  uint64_t flags)
446{
447	int i;
448
449	for (i = 0; i < nb_pkts; ++i, ++txdp, ++pkt)
450		vtx1(txdp, *pkt, flags);
451}
452
453uint16_t
454ixgbe_xmit_pkts_vec(void *tx_queue, struct rte_mbuf **tx_pkts,
455		       uint16_t nb_pkts)
456{
457	struct ixgbe_tx_queue *txq = (struct ixgbe_tx_queue *)tx_queue;
458	volatile union ixgbe_adv_tx_desc *txdp;
459	struct ixgbe_tx_entry_v *txep;
460	uint16_t n, nb_commit, tx_id;
461	uint64_t flags = DCMD_DTYP_FLAGS;
462	uint64_t rs = IXGBE_ADVTXD_DCMD_RS | DCMD_DTYP_FLAGS;
463	int i;
464
465	/* cross rx_thresh boundary is not allowed */
466	nb_pkts = RTE_MIN(nb_pkts, txq->tx_rs_thresh);
467
468	if (txq->nb_tx_free < txq->tx_free_thresh)
469		ixgbe_tx_free_bufs(txq);
470
471	nb_commit = nb_pkts = (uint16_t)RTE_MIN(txq->nb_tx_free, nb_pkts);
472	if (unlikely(nb_pkts == 0))
473		return 0;
474
475	tx_id = txq->tx_tail;
476	txdp = &txq->tx_ring[tx_id];
477	txep = &txq->sw_ring_v[tx_id];
478
479	txq->nb_tx_free = (uint16_t)(txq->nb_tx_free - nb_pkts);
480
481	n = (uint16_t)(txq->nb_tx_desc - tx_id);
482	if (nb_commit >= n) {
483		tx_backlog_entry(txep, tx_pkts, n);
484
485		for (i = 0; i < n - 1; ++i, ++tx_pkts, ++txdp)
486			vtx1(txdp, *tx_pkts, flags);
487
488		vtx1(txdp, *tx_pkts++, rs);
489
490		nb_commit = (uint16_t)(nb_commit - n);
491
492		tx_id = 0;
493		txq->tx_next_rs = (uint16_t)(txq->tx_rs_thresh - 1);
494
495		/* avoid reach the end of ring */
496		txdp = &txq->tx_ring[tx_id];
497		txep = &txq->sw_ring_v[tx_id];
498	}
499
500	tx_backlog_entry(txep, tx_pkts, nb_commit);
501
502	vtx(txdp, tx_pkts, nb_commit, flags);
503
504	tx_id = (uint16_t)(tx_id + nb_commit);
505	if (tx_id > txq->tx_next_rs) {
506		txq->tx_ring[txq->tx_next_rs].read.cmd_type_len |=
507			rte_cpu_to_le_32(IXGBE_ADVTXD_DCMD_RS);
508		txq->tx_next_rs = (uint16_t)(txq->tx_next_rs +
509			txq->tx_rs_thresh);
510	}
511
512	txq->tx_tail = tx_id;
513
514	IXGBE_PCI_REG_WRITE(txq->tdt_reg_addr, txq->tx_tail);
515
516	return nb_pkts;
517}
518
519static void __attribute__((cold))
520ixgbe_tx_queue_release_mbufs_vec(struct ixgbe_tx_queue *txq)
521{
522	_ixgbe_tx_queue_release_mbufs_vec(txq);
523}
524
525void __attribute__((cold))
526ixgbe_rx_queue_release_mbufs_vec(struct ixgbe_rx_queue *rxq)
527{
528	_ixgbe_rx_queue_release_mbufs_vec(rxq);
529}
530
531static void __attribute__((cold))
532ixgbe_tx_free_swring(struct ixgbe_tx_queue *txq)
533{
534	_ixgbe_tx_free_swring_vec(txq);
535}
536
537static void __attribute__((cold))
538ixgbe_reset_tx_queue(struct ixgbe_tx_queue *txq)
539{
540	_ixgbe_reset_tx_queue_vec(txq);
541}
542
543static const struct ixgbe_txq_ops vec_txq_ops = {
544	.release_mbufs = ixgbe_tx_queue_release_mbufs_vec,
545	.free_swring = ixgbe_tx_free_swring,
546	.reset = ixgbe_reset_tx_queue,
547};
548
549int __attribute__((cold))
550ixgbe_rxq_vec_setup(struct ixgbe_rx_queue *rxq)
551{
552	return ixgbe_rxq_vec_setup_default(rxq);
553}
554
555int __attribute__((cold))
556ixgbe_txq_vec_setup(struct ixgbe_tx_queue *txq)
557{
558	return ixgbe_txq_vec_setup_default(txq, &vec_txq_ops);
559}
560
561int __attribute__((cold))
562ixgbe_rx_vec_dev_conf_condition_check(struct rte_eth_dev *dev)
563{
564	struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
565
566	/* no csum error report support */
567	if (rxmode->hw_ip_checksum == 1)
568		return -1;
569
570	return ixgbe_rx_vec_dev_conf_condition_check_default(dev);
571}
572