197f17497SC.J. Collier/*
297f17497SC.J. Collier * Copyright (c) 2013-2015 Brocade Communications Systems, Inc.
397f17497SC.J. Collier *
497f17497SC.J. Collier * Copyright (c) 2015 QLogic Corporation.
597f17497SC.J. Collier * All rights reserved.
697f17497SC.J. Collier * www.qlogic.com
797f17497SC.J. Collier *
897f17497SC.J. Collier * See LICENSE.bnx2x_pmd for copyright and licensing details.
997f17497SC.J. Collier */
1097f17497SC.J. Collier
1197f17497SC.J. Collier#include "bnx2x.h"
1297f17497SC.J. Collier#include "bnx2x_rxtx.h"
1397f17497SC.J. Collier
1497f17497SC.J. Collierstatic const struct rte_memzone *
1597f17497SC.J. Collierring_dma_zone_reserve(struct rte_eth_dev *dev, const char *ring_name,
1697f17497SC.J. Collier		      uint16_t queue_id, uint32_t ring_size, int socket_id)
1797f17497SC.J. Collier{
1897f17497SC.J. Collier	char z_name[RTE_MEMZONE_NAMESIZE];
1997f17497SC.J. Collier	const struct rte_memzone *mz;
2097f17497SC.J. Collier
2197f17497SC.J. Collier	snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d",
226b3e017eSChristian Ehrhardt			dev->driver->pci_drv.driver.name, ring_name,
236b3e017eSChristian Ehrhardt			dev->data->port_id, queue_id);
2497f17497SC.J. Collier
2597f17497SC.J. Collier	mz = rte_memzone_lookup(z_name);
2697f17497SC.J. Collier	if (mz)
2797f17497SC.J. Collier		return mz;
2897f17497SC.J. Collier
2997f17497SC.J. Collier	return rte_memzone_reserve_aligned(z_name, ring_size, socket_id, 0, BNX2X_PAGE_SIZE);
3097f17497SC.J. Collier}
3197f17497SC.J. Collier
3297f17497SC.J. Collierstatic void
3397f17497SC.J. Collierbnx2x_rx_queue_release(struct bnx2x_rx_queue *rx_queue)
3497f17497SC.J. Collier{
3597f17497SC.J. Collier	uint16_t i;
3697f17497SC.J. Collier	struct rte_mbuf **sw_ring;
3797f17497SC.J. Collier
3897f17497SC.J. Collier	if (NULL != rx_queue) {
3997f17497SC.J. Collier
4097f17497SC.J. Collier		sw_ring = rx_queue->sw_ring;
4197f17497SC.J. Collier		if (NULL != sw_ring) {
4297f17497SC.J. Collier			for (i = 0; i < rx_queue->nb_rx_desc; i++) {
4397f17497SC.J. Collier				if (NULL != sw_ring[i])
4497f17497SC.J. Collier					rte_pktmbuf_free(sw_ring[i]);
4597f17497SC.J. Collier			}
4697f17497SC.J. Collier			rte_free(sw_ring);
4797f17497SC.J. Collier		}
4897f17497SC.J. Collier		rte_free(rx_queue);
4997f17497SC.J. Collier	}
5097f17497SC.J. Collier}
5197f17497SC.J. Collier
5297f17497SC.J. Colliervoid
5397f17497SC.J. Collierbnx2x_dev_rx_queue_release(void *rxq)
5497f17497SC.J. Collier{
5597f17497SC.J. Collier	bnx2x_rx_queue_release(rxq);
5697f17497SC.J. Collier}
5797f17497SC.J. Collier
5897f17497SC.J. Collierint
5997f17497SC.J. Collierbnx2x_dev_rx_queue_setup(struct rte_eth_dev *dev,
6097f17497SC.J. Collier		       uint16_t queue_idx,
6197f17497SC.J. Collier		       uint16_t nb_desc,
6297f17497SC.J. Collier		       unsigned int socket_id,
636b3e017eSChristian Ehrhardt		       __rte_unused const struct rte_eth_rxconf *rx_conf,
6497f17497SC.J. Collier		       struct rte_mempool *mp)
6597f17497SC.J. Collier{
6697f17497SC.J. Collier	uint16_t j, idx;
6797f17497SC.J. Collier	const struct rte_memzone *dma;
6897f17497SC.J. Collier	struct bnx2x_rx_queue *rxq;
6997f17497SC.J. Collier	uint32_t dma_size;
7097f17497SC.J. Collier	struct rte_mbuf *mbuf;
7197f17497SC.J. Collier	struct bnx2x_softc *sc = dev->data->dev_private;
7297f17497SC.J. Collier	struct bnx2x_fastpath *fp = &sc->fp[queue_idx];
7397f17497SC.J. Collier	struct eth_rx_cqe_next_page *nextpg;
7497f17497SC.J. Collier	phys_addr_t *rx_bd;
7597f17497SC.J. Collier	phys_addr_t busaddr;
7697f17497SC.J. Collier
7797f17497SC.J. Collier	/* First allocate the rx queue data structure */
7897f17497SC.J. Collier	rxq = rte_zmalloc_socket("ethdev RX queue", sizeof(struct bnx2x_rx_queue),
7997f17497SC.J. Collier				 RTE_CACHE_LINE_SIZE, socket_id);
8097f17497SC.J. Collier	if (NULL == rxq) {
81f7a9461eSLuca Boccassi		PMD_DRV_LOG(ERR, sc, "rte_zmalloc for rxq failed!");
8297f17497SC.J. Collier		return -ENOMEM;
8397f17497SC.J. Collier	}
8497f17497SC.J. Collier	rxq->sc = sc;
8597f17497SC.J. Collier	rxq->mb_pool = mp;
8697f17497SC.J. Collier	rxq->queue_id = queue_idx;
8797f17497SC.J. Collier	rxq->port_id = dev->data->port_id;
8897f17497SC.J. Collier
8997f17497SC.J. Collier	rxq->nb_rx_pages = 1;
9097f17497SC.J. Collier	while (USABLE_RX_BD(rxq) < nb_desc)
9197f17497SC.J. Collier		rxq->nb_rx_pages <<= 1;
9297f17497SC.J. Collier
9397f17497SC.J. Collier	rxq->nb_rx_desc  = TOTAL_RX_BD(rxq);
9497f17497SC.J. Collier	sc->rx_ring_size = USABLE_RX_BD(rxq);
9597f17497SC.J. Collier	rxq->nb_cq_pages = RCQ_BD_PAGES(rxq);
9697f17497SC.J. Collier
97f7a9461eSLuca Boccassi	PMD_DRV_LOG(DEBUG, sc, "fp[%02d] req_bd=%u, usable_bd=%lu, "
9897f17497SC.J. Collier		       "total_bd=%lu, rx_pages=%u, cq_pages=%u",
996b3e017eSChristian Ehrhardt		       queue_idx, nb_desc, (unsigned long)USABLE_RX_BD(rxq),
10097f17497SC.J. Collier		       (unsigned long)TOTAL_RX_BD(rxq), rxq->nb_rx_pages,
10197f17497SC.J. Collier		       rxq->nb_cq_pages);
10297f17497SC.J. Collier
10397f17497SC.J. Collier	/* Allocate RX ring hardware descriptors */
10497f17497SC.J. Collier	dma_size = rxq->nb_rx_desc * sizeof(struct eth_rx_bd);
10597f17497SC.J. Collier	dma = ring_dma_zone_reserve(dev, "hw_ring", queue_idx, dma_size, socket_id);
10697f17497SC.J. Collier	if (NULL == dma) {
10797f17497SC.J. Collier		PMD_RX_LOG(ERR, "ring_dma_zone_reserve for rx_ring failed!");
10897f17497SC.J. Collier		bnx2x_rx_queue_release(rxq);
10997f17497SC.J. Collier		return -ENOMEM;
11097f17497SC.J. Collier	}
11197f17497SC.J. Collier	fp->rx_desc_mapping = rxq->rx_ring_phys_addr = (uint64_t)dma->phys_addr;
11297f17497SC.J. Collier	rxq->rx_ring = (uint64_t*)dma->addr;
11397f17497SC.J. Collier	memset((void *)rxq->rx_ring, 0, dma_size);
11497f17497SC.J. Collier
11597f17497SC.J. Collier	/* Link the RX chain pages. */
11697f17497SC.J. Collier	for (j = 1; j <= rxq->nb_rx_pages; j++) {
11797f17497SC.J. Collier		rx_bd = &rxq->rx_ring[TOTAL_RX_BD_PER_PAGE * j - 2];
11897f17497SC.J. Collier		busaddr = rxq->rx_ring_phys_addr + BNX2X_PAGE_SIZE * (j % rxq->nb_rx_pages);
11997f17497SC.J. Collier		*rx_bd = busaddr;
12097f17497SC.J. Collier	}
12197f17497SC.J. Collier
12297f17497SC.J. Collier	/* Allocate software ring */
12397f17497SC.J. Collier	dma_size = rxq->nb_rx_desc * sizeof(struct bnx2x_rx_entry);
12497f17497SC.J. Collier	rxq->sw_ring = rte_zmalloc_socket("sw_ring", dma_size,
12597f17497SC.J. Collier					  RTE_CACHE_LINE_SIZE,
12697f17497SC.J. Collier					  socket_id);
12797f17497SC.J. Collier	if (NULL == rxq->sw_ring) {
12897f17497SC.J. Collier		PMD_RX_LOG(ERR, "rte_zmalloc for sw_ring failed!");
12997f17497SC.J. Collier		bnx2x_rx_queue_release(rxq);
13097f17497SC.J. Collier		return -ENOMEM;
13197f17497SC.J. Collier	}
13297f17497SC.J. Collier
13397f17497SC.J. Collier	/* Initialize software ring entries */
13497f17497SC.J. Collier	for (idx = 0; idx < rxq->nb_rx_desc; idx = NEXT_RX_BD(idx)) {
1358b25d1adSChristian Ehrhardt		mbuf = rte_mbuf_raw_alloc(mp);
13697f17497SC.J. Collier		if (NULL == mbuf) {
13797f17497SC.J. Collier			PMD_RX_LOG(ERR, "RX mbuf alloc failed queue_id=%u, idx=%d",
13897f17497SC.J. Collier				   (unsigned)rxq->queue_id, idx);
13997f17497SC.J. Collier			bnx2x_rx_queue_release(rxq);
14097f17497SC.J. Collier			return -ENOMEM;
14197f17497SC.J. Collier		}
14297f17497SC.J. Collier		rxq->sw_ring[idx] = mbuf;
14397f17497SC.J. Collier		rxq->rx_ring[idx] = mbuf->buf_physaddr;
14497f17497SC.J. Collier	}
14597f17497SC.J. Collier	rxq->pkt_first_seg = NULL;
14697f17497SC.J. Collier	rxq->pkt_last_seg = NULL;
14797f17497SC.J. Collier	rxq->rx_bd_head = 0;
14897f17497SC.J. Collier	rxq->rx_bd_tail = rxq->nb_rx_desc;
14997f17497SC.J. Collier
15097f17497SC.J. Collier	/* Allocate CQ chain. */
15197f17497SC.J. Collier	dma_size = BNX2X_RX_CHAIN_PAGE_SZ * rxq->nb_cq_pages;
15297f17497SC.J. Collier	dma = ring_dma_zone_reserve(dev, "bnx2x_rcq", queue_idx, dma_size, socket_id);
15397f17497SC.J. Collier	if (NULL == dma) {
15497f17497SC.J. Collier		PMD_RX_LOG(ERR, "RCQ  alloc failed");
15597f17497SC.J. Collier		return -ENOMEM;
15697f17497SC.J. Collier	}
15797f17497SC.J. Collier	fp->rx_comp_mapping = rxq->cq_ring_phys_addr = (uint64_t)dma->phys_addr;
15897f17497SC.J. Collier	rxq->cq_ring = (union eth_rx_cqe*)dma->addr;
15997f17497SC.J. Collier
16097f17497SC.J. Collier	/* Link the CQ chain pages. */
16197f17497SC.J. Collier	for (j = 1; j <= rxq->nb_cq_pages; j++) {
16297f17497SC.J. Collier		nextpg = &rxq->cq_ring[TOTAL_RCQ_ENTRIES_PER_PAGE * j - 1].next_page_cqe;
16397f17497SC.J. Collier		busaddr = rxq->cq_ring_phys_addr + BNX2X_PAGE_SIZE * (j % rxq->nb_cq_pages);
16497f17497SC.J. Collier		nextpg->addr_hi = rte_cpu_to_le_32(U64_HI(busaddr));
16597f17497SC.J. Collier		nextpg->addr_lo = rte_cpu_to_le_32(U64_LO(busaddr));
16697f17497SC.J. Collier	}
16797f17497SC.J. Collier	rxq->rx_cq_head = 0;
16897f17497SC.J. Collier	rxq->rx_cq_tail = TOTAL_RCQ_ENTRIES(rxq);
16997f17497SC.J. Collier
17097f17497SC.J. Collier	dev->data->rx_queues[queue_idx] = rxq;
17197f17497SC.J. Collier	if (!sc->rx_queues) sc->rx_queues = dev->data->rx_queues;
17297f17497SC.J. Collier
17397f17497SC.J. Collier	return 0;
17497f17497SC.J. Collier}
17597f17497SC.J. Collier
17697f17497SC.J. Collierstatic void
17797f17497SC.J. Collierbnx2x_tx_queue_release(struct bnx2x_tx_queue *tx_queue)
17897f17497SC.J. Collier{
17997f17497SC.J. Collier	uint16_t i;
18097f17497SC.J. Collier	struct rte_mbuf **sw_ring;
18197f17497SC.J. Collier
18297f17497SC.J. Collier	if (NULL != tx_queue) {
18397f17497SC.J. Collier
18497f17497SC.J. Collier		sw_ring = tx_queue->sw_ring;
18597f17497SC.J. Collier		if (NULL != sw_ring) {
18697f17497SC.J. Collier			for (i = 0; i < tx_queue->nb_tx_desc; i++) {
18797f17497SC.J. Collier				if (NULL != sw_ring[i])
18897f17497SC.J. Collier					rte_pktmbuf_free(sw_ring[i]);
18997f17497SC.J. Collier			}
19097f17497SC.J. Collier			rte_free(sw_ring);
19197f17497SC.J. Collier		}
19297f17497SC.J. Collier		rte_free(tx_queue);
19397f17497SC.J. Collier	}
19497f17497SC.J. Collier}
19597f17497SC.J. Collier
19697f17497SC.J. Colliervoid
19797f17497SC.J. Collierbnx2x_dev_tx_queue_release(void *txq)
19897f17497SC.J. Collier{
19997f17497SC.J. Collier	bnx2x_tx_queue_release(txq);
20097f17497SC.J. Collier}
20197f17497SC.J. Collier
20297f17497SC.J. Collierstatic uint16_t
20397f17497SC.J. Collierbnx2x_xmit_pkts(void *p_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
20497f17497SC.J. Collier{
20597f17497SC.J. Collier	struct bnx2x_tx_queue *txq;
20697f17497SC.J. Collier	struct bnx2x_softc *sc;
20797f17497SC.J. Collier	struct bnx2x_fastpath *fp;
2088b25d1adSChristian Ehrhardt	uint16_t nb_tx_pkts;
2098b25d1adSChristian Ehrhardt	uint16_t nb_pkt_sent = 0;
2108b25d1adSChristian Ehrhardt	uint32_t ret;
21197f17497SC.J. Collier
21297f17497SC.J. Collier	txq = p_txq;
21397f17497SC.J. Collier	sc = txq->sc;
21497f17497SC.J. Collier	fp = &sc->fp[txq->queue_id];
21597f17497SC.J. Collier
2168b25d1adSChristian Ehrhardt	if ((unlikely((txq->nb_tx_desc - txq->nb_tx_avail) >
2178b25d1adSChristian Ehrhardt				txq->tx_free_thresh)))
2188b25d1adSChristian Ehrhardt		bnx2x_txeof(sc, fp);
21997f17497SC.J. Collier
2208b25d1adSChristian Ehrhardt	nb_tx_pkts = RTE_MIN(nb_pkts, txq->nb_tx_avail / BDS_PER_TX_PKT);
2218b25d1adSChristian Ehrhardt	if (unlikely(nb_tx_pkts == 0))
2228b25d1adSChristian Ehrhardt		return 0;
22397f17497SC.J. Collier
2248b25d1adSChristian Ehrhardt	while (nb_tx_pkts--) {
2258b25d1adSChristian Ehrhardt		struct rte_mbuf *m = *tx_pkts++;
2268b25d1adSChristian Ehrhardt		assert(m != NULL);
2278b25d1adSChristian Ehrhardt		ret = bnx2x_tx_encap(txq, m);
2288b25d1adSChristian Ehrhardt		fp->tx_db.data.prod += ret;
2298b25d1adSChristian Ehrhardt		nb_pkt_sent++;
2308b25d1adSChristian Ehrhardt	}
23197f17497SC.J. Collier
2328b25d1adSChristian Ehrhardt	bnx2x_update_fp_sb_idx(fp);
2338b25d1adSChristian Ehrhardt	mb();
2348b25d1adSChristian Ehrhardt	DOORBELL(sc, txq->queue_id, fp->tx_db.raw);
2358b25d1adSChristian Ehrhardt	mb();
23697f17497SC.J. Collier
2378b25d1adSChristian Ehrhardt	if ((txq->nb_tx_desc - txq->nb_tx_avail) >
2388b25d1adSChristian Ehrhardt				txq->tx_free_thresh)
2398b25d1adSChristian Ehrhardt		bnx2x_txeof(sc, fp);
24097f17497SC.J. Collier
2418b25d1adSChristian Ehrhardt	return nb_pkt_sent;
24297f17497SC.J. Collier}
24397f17497SC.J. Collier
24497f17497SC.J. Collierint
24597f17497SC.J. Collierbnx2x_dev_tx_queue_setup(struct rte_eth_dev *dev,
24697f17497SC.J. Collier		       uint16_t queue_idx,
24797f17497SC.J. Collier		       uint16_t nb_desc,
24897f17497SC.J. Collier		       unsigned int socket_id,
24997f17497SC.J. Collier		       const struct rte_eth_txconf *tx_conf)
25097f17497SC.J. Collier{
25197f17497SC.J. Collier	uint16_t i;
25297f17497SC.J. Collier	unsigned int tsize;
25397f17497SC.J. Collier	const struct rte_memzone *tz;
25497f17497SC.J. Collier	struct bnx2x_tx_queue *txq;
25597f17497SC.J. Collier	struct eth_tx_next_bd *tx_n_bd;
25697f17497SC.J. Collier	uint64_t busaddr;
25797f17497SC.J. Collier	struct bnx2x_softc *sc = dev->data->dev_private;
25897f17497SC.J. Collier	struct bnx2x_fastpath *fp = &sc->fp[queue_idx];
25997f17497SC.J. Collier
26097f17497SC.J. Collier	/* First allocate the tx queue data structure */
26197f17497SC.J. Collier	txq = rte_zmalloc("ethdev TX queue", sizeof(struct bnx2x_tx_queue),
26297f17497SC.J. Collier			  RTE_CACHE_LINE_SIZE);
26397f17497SC.J. Collier	if (txq == NULL)
26497f17497SC.J. Collier		return -ENOMEM;
26597f17497SC.J. Collier	txq->sc = sc;
26697f17497SC.J. Collier
26797f17497SC.J. Collier	txq->nb_tx_pages = 1;
26897f17497SC.J. Collier	while (USABLE_TX_BD(txq) < nb_desc)
26997f17497SC.J. Collier		txq->nb_tx_pages <<= 1;
27097f17497SC.J. Collier
27197f17497SC.J. Collier	txq->nb_tx_desc  = TOTAL_TX_BD(txq);
27297f17497SC.J. Collier	sc->tx_ring_size = TOTAL_TX_BD(txq);
27397f17497SC.J. Collier
27497f17497SC.J. Collier	txq->tx_free_thresh = tx_conf->tx_free_thresh ?
27597f17497SC.J. Collier		tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH;
276aab0c291SChristian Ehrhardt	txq->tx_free_thresh = min(txq->tx_free_thresh,
277aab0c291SChristian Ehrhardt				  txq->nb_tx_desc - BDS_PER_TX_PKT);
27897f17497SC.J. Collier
279f7a9461eSLuca Boccassi	PMD_DRV_LOG(DEBUG, sc, "fp[%02d] req_bd=%u, thresh=%u, usable_bd=%lu, "
28097f17497SC.J. Collier		     "total_bd=%lu, tx_pages=%u",
28197f17497SC.J. Collier		     queue_idx, nb_desc, txq->tx_free_thresh,
28297f17497SC.J. Collier		     (unsigned long)USABLE_TX_BD(txq),
28397f17497SC.J. Collier		     (unsigned long)TOTAL_TX_BD(txq), txq->nb_tx_pages);
28497f17497SC.J. Collier
28597f17497SC.J. Collier	/* Allocate TX ring hardware descriptors */
28697f17497SC.J. Collier	tsize = txq->nb_tx_desc * sizeof(union eth_tx_bd_types);
28797f17497SC.J. Collier	tz = ring_dma_zone_reserve(dev, "tx_hw_ring", queue_idx, tsize, socket_id);
28897f17497SC.J. Collier	if (tz == NULL) {
28997f17497SC.J. Collier		bnx2x_tx_queue_release(txq);
29097f17497SC.J. Collier		return -ENOMEM;
29197f17497SC.J. Collier	}
29297f17497SC.J. Collier	fp->tx_desc_mapping = txq->tx_ring_phys_addr = (uint64_t)tz->phys_addr;
29397f17497SC.J. Collier	txq->tx_ring = (union eth_tx_bd_types *) tz->addr;
29497f17497SC.J. Collier	memset(txq->tx_ring, 0, tsize);
29597f17497SC.J. Collier
29697f17497SC.J. Collier	/* Allocate software ring */
29797f17497SC.J. Collier	tsize = txq->nb_tx_desc * sizeof(struct rte_mbuf *);
29897f17497SC.J. Collier	txq->sw_ring = rte_zmalloc("tx_sw_ring", tsize,
29997f17497SC.J. Collier				   RTE_CACHE_LINE_SIZE);
30097f17497SC.J. Collier	if (txq->sw_ring == NULL) {
30197f17497SC.J. Collier		bnx2x_tx_queue_release(txq);
30297f17497SC.J. Collier		return -ENOMEM;
30397f17497SC.J. Collier	}
30497f17497SC.J. Collier
305f7a9461eSLuca Boccassi	/* PMD_DRV_LOG(DEBUG, sc, "sw_ring=%p hw_ring=%p dma_addr=0x%"PRIx64,
30697f17497SC.J. Collier	   txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr); */
30797f17497SC.J. Collier
30897f17497SC.J. Collier	/* Link TX pages */
30997f17497SC.J. Collier	for (i = 1; i <= txq->nb_tx_pages; i++) {
31097f17497SC.J. Collier		tx_n_bd = &txq->tx_ring[TOTAL_TX_BD_PER_PAGE * i - 1].next_bd;
31197f17497SC.J. Collier		busaddr = txq->tx_ring_phys_addr + BNX2X_PAGE_SIZE * (i % txq->nb_tx_pages);
31297f17497SC.J. Collier		tx_n_bd->addr_hi = rte_cpu_to_le_32(U64_HI(busaddr));
31397f17497SC.J. Collier		tx_n_bd->addr_lo = rte_cpu_to_le_32(U64_LO(busaddr));
314f7a9461eSLuca Boccassi		/* PMD_DRV_LOG(DEBUG, sc, "link tx page %lu",
315f7a9461eSLuca Boccassi		 *          (TOTAL_TX_BD_PER_PAGE * i - 1));
316f7a9461eSLuca Boccassi		 */
31797f17497SC.J. Collier	}
31897f17497SC.J. Collier
31997f17497SC.J. Collier	txq->queue_id = queue_idx;
32097f17497SC.J. Collier	txq->port_id = dev->data->port_id;
32197f17497SC.J. Collier	txq->tx_pkt_tail = 0;
32297f17497SC.J. Collier	txq->tx_pkt_head = 0;
32397f17497SC.J. Collier	txq->tx_bd_tail = 0;
32497f17497SC.J. Collier	txq->tx_bd_head = 0;
32597f17497SC.J. Collier	txq->nb_tx_avail = txq->nb_tx_desc;
32697f17497SC.J. Collier	dev->tx_pkt_burst = bnx2x_xmit_pkts;
32797f17497SC.J. Collier	dev->data->tx_queues[queue_idx] = txq;
32897f17497SC.J. Collier	if (!sc->tx_queues) sc->tx_queues = dev->data->tx_queues;
32997f17497SC.J. Collier
33097f17497SC.J. Collier	return 0;
33197f17497SC.J. Collier}
33297f17497SC.J. Collier
33397f17497SC.J. Collierstatic inline void
33497f17497SC.J. Collierbnx2x_upd_rx_prod_fast(struct bnx2x_softc *sc, struct bnx2x_fastpath *fp,
33597f17497SC.J. Collier		uint16_t rx_bd_prod, uint16_t rx_cq_prod)
33697f17497SC.J. Collier{
33797f17497SC.J. Collier	union ustorm_eth_rx_producers rx_prods;
33897f17497SC.J. Collier
33997f17497SC.J. Collier	rx_prods.prod.bd_prod  = rx_bd_prod;
34097f17497SC.J. Collier	rx_prods.prod.cqe_prod = rx_cq_prod;
34197f17497SC.J. Collier
34297f17497SC.J. Collier	REG_WR(sc, fp->ustorm_rx_prods_offset, rx_prods.raw_data[0]);
34397f17497SC.J. Collier}
34497f17497SC.J. Collier
34597f17497SC.J. Collierstatic uint16_t
34697f17497SC.J. Collierbnx2x_recv_pkts(void *p_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
34797f17497SC.J. Collier{
34897f17497SC.J. Collier	struct bnx2x_rx_queue *rxq = p_rxq;
34997f17497SC.J. Collier	struct bnx2x_softc *sc = rxq->sc;
35097f17497SC.J. Collier	struct bnx2x_fastpath *fp = &sc->fp[rxq->queue_id];
35197f17497SC.J. Collier	uint32_t nb_rx = 0;
35297f17497SC.J. Collier	uint16_t hw_cq_cons, sw_cq_cons, sw_cq_prod;
35397f17497SC.J. Collier	uint16_t bd_cons, bd_prod;
35497f17497SC.J. Collier	struct rte_mbuf *new_mb;
35597f17497SC.J. Collier	uint16_t rx_pref;
35697f17497SC.J. Collier	struct eth_fast_path_rx_cqe *cqe_fp;
35797f17497SC.J. Collier	uint16_t len, pad;
35897f17497SC.J. Collier	struct rte_mbuf *rx_mb = NULL;
35997f17497SC.J. Collier
36097f17497SC.J. Collier	hw_cq_cons = le16toh(*fp->rx_cq_cons_sb);
36197f17497SC.J. Collier	if ((hw_cq_cons & USABLE_RCQ_ENTRIES_PER_PAGE) ==
36297f17497SC.J. Collier			USABLE_RCQ_ENTRIES_PER_PAGE) {
36397f17497SC.J. Collier		++hw_cq_cons;
36497f17497SC.J. Collier	}
36597f17497SC.J. Collier
36697f17497SC.J. Collier	bd_cons = rxq->rx_bd_head;
36797f17497SC.J. Collier	bd_prod = rxq->rx_bd_tail;
36897f17497SC.J. Collier	sw_cq_cons = rxq->rx_cq_head;
36997f17497SC.J. Collier	sw_cq_prod = rxq->rx_cq_tail;
37097f17497SC.J. Collier
37197f17497SC.J. Collier	if (sw_cq_cons == hw_cq_cons)
37297f17497SC.J. Collier		return 0;
37397f17497SC.J. Collier
37497f17497SC.J. Collier	while (nb_rx < nb_pkts && sw_cq_cons != hw_cq_cons) {
37597f17497SC.J. Collier
37697f17497SC.J. Collier		bd_prod &= MAX_RX_BD(rxq);
37797f17497SC.J. Collier		bd_cons &= MAX_RX_BD(rxq);
37897f17497SC.J. Collier
37997f17497SC.J. Collier		cqe_fp = &rxq->cq_ring[sw_cq_cons & MAX_RX_BD(rxq)].fast_path_cqe;
38097f17497SC.J. Collier
38197f17497SC.J. Collier		if (unlikely(CQE_TYPE_SLOW(cqe_fp->type_error_flags & ETH_FAST_PATH_RX_CQE_TYPE))) {
38297f17497SC.J. Collier			PMD_RX_LOG(ERR, "slowpath event during traffic processing");
38397f17497SC.J. Collier			break;
38497f17497SC.J. Collier		}
38597f17497SC.J. Collier
38697f17497SC.J. Collier		if (unlikely(cqe_fp->type_error_flags & ETH_FAST_PATH_RX_CQE_PHY_DECODE_ERR_FLG)) {
38797f17497SC.J. Collier			PMD_RX_LOG(ERR, "flags 0x%x rx packet %u",
38897f17497SC.J. Collier					cqe_fp->type_error_flags, sw_cq_cons);
38997f17497SC.J. Collier			goto next_rx;
39097f17497SC.J. Collier		}
39197f17497SC.J. Collier
39297f17497SC.J. Collier		len = cqe_fp->pkt_len_or_gro_seg_len;
39397f17497SC.J. Collier		pad = cqe_fp->placement_offset;
39497f17497SC.J. Collier
3958b25d1adSChristian Ehrhardt		new_mb = rte_mbuf_raw_alloc(rxq->mb_pool);
39697f17497SC.J. Collier		if (unlikely(!new_mb)) {
39797f17497SC.J. Collier			PMD_RX_LOG(ERR, "mbuf alloc fail fp[%02d]", fp->index);
3988b25d1adSChristian Ehrhardt			rte_eth_devices[rxq->port_id].data->
3998b25d1adSChristian Ehrhardt					rx_mbuf_alloc_failed++;
40097f17497SC.J. Collier			goto next_rx;
40197f17497SC.J. Collier		}
40297f17497SC.J. Collier
40397f17497SC.J. Collier		rx_mb = rxq->sw_ring[bd_cons];
40497f17497SC.J. Collier		rxq->sw_ring[bd_cons] = new_mb;
40597f17497SC.J. Collier		rxq->rx_ring[bd_prod] = new_mb->buf_physaddr;
40697f17497SC.J. Collier
40797f17497SC.J. Collier		rx_pref = NEXT_RX_BD(bd_cons) & MAX_RX_BD(rxq);
40897f17497SC.J. Collier		rte_prefetch0(rxq->sw_ring[rx_pref]);
40997f17497SC.J. Collier		if ((rx_pref & 0x3) == 0) {
41097f17497SC.J. Collier			rte_prefetch0(&rxq->rx_ring[rx_pref]);
41197f17497SC.J. Collier			rte_prefetch0(&rxq->sw_ring[rx_pref]);
41297f17497SC.J. Collier		}
41397f17497SC.J. Collier
41497f17497SC.J. Collier		rx_mb->data_off = pad;
41597f17497SC.J. Collier		rx_mb->nb_segs = 1;
41697f17497SC.J. Collier		rx_mb->next = NULL;
41797f17497SC.J. Collier		rx_mb->pkt_len = rx_mb->data_len = len;
41897f17497SC.J. Collier		rx_mb->port = rxq->port_id;
41997f17497SC.J. Collier		rte_prefetch1(rte_pktmbuf_mtod(rx_mb, void *));
42097f17497SC.J. Collier
42197f17497SC.J. Collier		/*
42297f17497SC.J. Collier		 * If we received a packet with a vlan tag,
42397f17497SC.J. Collier		 * attach that information to the packet.
42497f17497SC.J. Collier		 */
42597f17497SC.J. Collier		if (cqe_fp->pars_flags.flags & PARSING_FLAGS_VLAN) {
42697f17497SC.J. Collier			rx_mb->vlan_tci = cqe_fp->vlan_tag;
42797f17497SC.J. Collier			rx_mb->ol_flags |= PKT_RX_VLAN_PKT;
42897f17497SC.J. Collier		}
42997f17497SC.J. Collier
43097f17497SC.J. Collier		rx_pkts[nb_rx] = rx_mb;
43197f17497SC.J. Collier		nb_rx++;
43297f17497SC.J. Collier
43397f17497SC.J. Collier		/* limit spinning on the queue */
43497f17497SC.J. Collier		if (unlikely(nb_rx == sc->rx_budget)) {
43597f17497SC.J. Collier			PMD_RX_LOG(ERR, "Limit spinning on the queue");
43697f17497SC.J. Collier			break;
43797f17497SC.J. Collier		}
43897f17497SC.J. Collier
43997f17497SC.J. Colliernext_rx:
44097f17497SC.J. Collier		bd_cons    = NEXT_RX_BD(bd_cons);
44197f17497SC.J. Collier		bd_prod    = NEXT_RX_BD(bd_prod);
44297f17497SC.J. Collier		sw_cq_prod = NEXT_RCQ_IDX(sw_cq_prod);
44397f17497SC.J. Collier		sw_cq_cons = NEXT_RCQ_IDX(sw_cq_cons);
44497f17497SC.J. Collier	}
44597f17497SC.J. Collier	rxq->rx_bd_head = bd_cons;
44697f17497SC.J. Collier	rxq->rx_bd_tail = bd_prod;
44797f17497SC.J. Collier	rxq->rx_cq_head = sw_cq_cons;
44897f17497SC.J. Collier	rxq->rx_cq_tail = sw_cq_prod;
44997f17497SC.J. Collier
45097f17497SC.J. Collier	bnx2x_upd_rx_prod_fast(sc, fp, bd_prod, sw_cq_prod);
45197f17497SC.J. Collier
45297f17497SC.J. Collier	return nb_rx;
45397f17497SC.J. Collier}
45497f17497SC.J. Collier
45597f17497SC.J. Collierint
45697f17497SC.J. Collierbnx2x_dev_rx_init(struct rte_eth_dev *dev)
45797f17497SC.J. Collier{
45897f17497SC.J. Collier	dev->rx_pkt_burst = bnx2x_recv_pkts;
45997f17497SC.J. Collier
46097f17497SC.J. Collier	return 0;
46197f17497SC.J. Collier}
46297f17497SC.J. Collier
46397f17497SC.J. Colliervoid
46497f17497SC.J. Collierbnx2x_dev_clear_queues(struct rte_eth_dev *dev)
46597f17497SC.J. Collier{
466f7a9461eSLuca Boccassi	struct bnx2x_softc *sc = dev->data->dev_private;
46797f17497SC.J. Collier	uint8_t i;
46897f17497SC.J. Collier
469f7a9461eSLuca Boccassi	PMD_INIT_FUNC_TRACE(sc);
47097f17497SC.J. Collier
47197f17497SC.J. Collier	for (i = 0; i < dev->data->nb_tx_queues; i++) {
47297f17497SC.J. Collier		struct bnx2x_tx_queue *txq = dev->data->tx_queues[i];
47397f17497SC.J. Collier		if (txq != NULL) {
47497f17497SC.J. Collier			bnx2x_tx_queue_release(txq);
47597f17497SC.J. Collier			dev->data->tx_queues[i] = NULL;
47697f17497SC.J. Collier		}
47797f17497SC.J. Collier	}
47897f17497SC.J. Collier
47997f17497SC.J. Collier	for (i = 0; i < dev->data->nb_rx_queues; i++) {
48097f17497SC.J. Collier		struct bnx2x_rx_queue *rxq = dev->data->rx_queues[i];
48197f17497SC.J. Collier		if (rxq != NULL) {
48297f17497SC.J. Collier			bnx2x_rx_queue_release(rxq);
48397f17497SC.J. Collier			dev->data->rx_queues[i] = NULL;
48497f17497SC.J. Collier		}
48597f17497SC.J. Collier	}
48697f17497SC.J. Collier}
487