1/*
2 * Copyright (c) 2013-2015 Brocade Communications Systems, Inc.
3 *
4 * Copyright (c) 2015 QLogic Corporation.
5 * All rights reserved.
6 * www.qlogic.com
7 *
8 * See LICENSE.bnx2x_pmd for copyright and licensing details.
9 */
10
11#include "bnx2x.h"
12#include "bnx2x_rxtx.h"
13
14static const struct rte_memzone *
15ring_dma_zone_reserve(struct rte_eth_dev *dev, const char *ring_name,
16		      uint16_t queue_id, uint32_t ring_size, int socket_id)
17{
18	char z_name[RTE_MEMZONE_NAMESIZE];
19	const struct rte_memzone *mz;
20
21	snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d",
22			dev->driver->pci_drv.driver.name, ring_name,
23			dev->data->port_id, queue_id);
24
25	mz = rte_memzone_lookup(z_name);
26	if (mz)
27		return mz;
28
29	return rte_memzone_reserve_aligned(z_name, ring_size, socket_id, 0, BNX2X_PAGE_SIZE);
30}
31
32static void
33bnx2x_rx_queue_release(struct bnx2x_rx_queue *rx_queue)
34{
35	uint16_t i;
36	struct rte_mbuf **sw_ring;
37
38	if (NULL != rx_queue) {
39
40		sw_ring = rx_queue->sw_ring;
41		if (NULL != sw_ring) {
42			for (i = 0; i < rx_queue->nb_rx_desc; i++) {
43				if (NULL != sw_ring[i])
44					rte_pktmbuf_free(sw_ring[i]);
45			}
46			rte_free(sw_ring);
47		}
48		rte_free(rx_queue);
49	}
50}
51
52void
53bnx2x_dev_rx_queue_release(void *rxq)
54{
55	bnx2x_rx_queue_release(rxq);
56}
57
58int
59bnx2x_dev_rx_queue_setup(struct rte_eth_dev *dev,
60		       uint16_t queue_idx,
61		       uint16_t nb_desc,
62		       unsigned int socket_id,
63		       __rte_unused const struct rte_eth_rxconf *rx_conf,
64		       struct rte_mempool *mp)
65{
66	uint16_t j, idx;
67	const struct rte_memzone *dma;
68	struct bnx2x_rx_queue *rxq;
69	uint32_t dma_size;
70	struct rte_mbuf *mbuf;
71	struct bnx2x_softc *sc = dev->data->dev_private;
72	struct bnx2x_fastpath *fp = &sc->fp[queue_idx];
73	struct eth_rx_cqe_next_page *nextpg;
74	phys_addr_t *rx_bd;
75	phys_addr_t busaddr;
76
77	/* First allocate the rx queue data structure */
78	rxq = rte_zmalloc_socket("ethdev RX queue", sizeof(struct bnx2x_rx_queue),
79				 RTE_CACHE_LINE_SIZE, socket_id);
80	if (NULL == rxq) {
81		PMD_DRV_LOG(ERR, sc, "rte_zmalloc for rxq failed!");
82		return -ENOMEM;
83	}
84	rxq->sc = sc;
85	rxq->mb_pool = mp;
86	rxq->queue_id = queue_idx;
87	rxq->port_id = dev->data->port_id;
88
89	rxq->nb_rx_pages = 1;
90	while (USABLE_RX_BD(rxq) < nb_desc)
91		rxq->nb_rx_pages <<= 1;
92
93	rxq->nb_rx_desc  = TOTAL_RX_BD(rxq);
94	sc->rx_ring_size = USABLE_RX_BD(rxq);
95	rxq->nb_cq_pages = RCQ_BD_PAGES(rxq);
96
97	PMD_DRV_LOG(DEBUG, sc, "fp[%02d] req_bd=%u, usable_bd=%lu, "
98		       "total_bd=%lu, rx_pages=%u, cq_pages=%u",
99		       queue_idx, nb_desc, (unsigned long)USABLE_RX_BD(rxq),
100		       (unsigned long)TOTAL_RX_BD(rxq), rxq->nb_rx_pages,
101		       rxq->nb_cq_pages);
102
103	/* Allocate RX ring hardware descriptors */
104	dma_size = rxq->nb_rx_desc * sizeof(struct eth_rx_bd);
105	dma = ring_dma_zone_reserve(dev, "hw_ring", queue_idx, dma_size, socket_id);
106	if (NULL == dma) {
107		PMD_RX_LOG(ERR, "ring_dma_zone_reserve for rx_ring failed!");
108		bnx2x_rx_queue_release(rxq);
109		return -ENOMEM;
110	}
111	fp->rx_desc_mapping = rxq->rx_ring_phys_addr = (uint64_t)dma->phys_addr;
112	rxq->rx_ring = (uint64_t*)dma->addr;
113	memset((void *)rxq->rx_ring, 0, dma_size);
114
115	/* Link the RX chain pages. */
116	for (j = 1; j <= rxq->nb_rx_pages; j++) {
117		rx_bd = &rxq->rx_ring[TOTAL_RX_BD_PER_PAGE * j - 2];
118		busaddr = rxq->rx_ring_phys_addr + BNX2X_PAGE_SIZE * (j % rxq->nb_rx_pages);
119		*rx_bd = busaddr;
120	}
121
122	/* Allocate software ring */
123	dma_size = rxq->nb_rx_desc * sizeof(struct bnx2x_rx_entry);
124	rxq->sw_ring = rte_zmalloc_socket("sw_ring", dma_size,
125					  RTE_CACHE_LINE_SIZE,
126					  socket_id);
127	if (NULL == rxq->sw_ring) {
128		PMD_RX_LOG(ERR, "rte_zmalloc for sw_ring failed!");
129		bnx2x_rx_queue_release(rxq);
130		return -ENOMEM;
131	}
132
133	/* Initialize software ring entries */
134	for (idx = 0; idx < rxq->nb_rx_desc; idx = NEXT_RX_BD(idx)) {
135		mbuf = rte_mbuf_raw_alloc(mp);
136		if (NULL == mbuf) {
137			PMD_RX_LOG(ERR, "RX mbuf alloc failed queue_id=%u, idx=%d",
138				   (unsigned)rxq->queue_id, idx);
139			bnx2x_rx_queue_release(rxq);
140			return -ENOMEM;
141		}
142		rxq->sw_ring[idx] = mbuf;
143		rxq->rx_ring[idx] = mbuf->buf_physaddr;
144	}
145	rxq->pkt_first_seg = NULL;
146	rxq->pkt_last_seg = NULL;
147	rxq->rx_bd_head = 0;
148	rxq->rx_bd_tail = rxq->nb_rx_desc;
149
150	/* Allocate CQ chain. */
151	dma_size = BNX2X_RX_CHAIN_PAGE_SZ * rxq->nb_cq_pages;
152	dma = ring_dma_zone_reserve(dev, "bnx2x_rcq", queue_idx, dma_size, socket_id);
153	if (NULL == dma) {
154		PMD_RX_LOG(ERR, "RCQ  alloc failed");
155		return -ENOMEM;
156	}
157	fp->rx_comp_mapping = rxq->cq_ring_phys_addr = (uint64_t)dma->phys_addr;
158	rxq->cq_ring = (union eth_rx_cqe*)dma->addr;
159
160	/* Link the CQ chain pages. */
161	for (j = 1; j <= rxq->nb_cq_pages; j++) {
162		nextpg = &rxq->cq_ring[TOTAL_RCQ_ENTRIES_PER_PAGE * j - 1].next_page_cqe;
163		busaddr = rxq->cq_ring_phys_addr + BNX2X_PAGE_SIZE * (j % rxq->nb_cq_pages);
164		nextpg->addr_hi = rte_cpu_to_le_32(U64_HI(busaddr));
165		nextpg->addr_lo = rte_cpu_to_le_32(U64_LO(busaddr));
166	}
167	rxq->rx_cq_head = 0;
168	rxq->rx_cq_tail = TOTAL_RCQ_ENTRIES(rxq);
169
170	dev->data->rx_queues[queue_idx] = rxq;
171	if (!sc->rx_queues) sc->rx_queues = dev->data->rx_queues;
172
173	return 0;
174}
175
176static void
177bnx2x_tx_queue_release(struct bnx2x_tx_queue *tx_queue)
178{
179	uint16_t i;
180	struct rte_mbuf **sw_ring;
181
182	if (NULL != tx_queue) {
183
184		sw_ring = tx_queue->sw_ring;
185		if (NULL != sw_ring) {
186			for (i = 0; i < tx_queue->nb_tx_desc; i++) {
187				if (NULL != sw_ring[i])
188					rte_pktmbuf_free(sw_ring[i]);
189			}
190			rte_free(sw_ring);
191		}
192		rte_free(tx_queue);
193	}
194}
195
196void
197bnx2x_dev_tx_queue_release(void *txq)
198{
199	bnx2x_tx_queue_release(txq);
200}
201
202static uint16_t
203bnx2x_xmit_pkts(void *p_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
204{
205	struct bnx2x_tx_queue *txq;
206	struct bnx2x_softc *sc;
207	struct bnx2x_fastpath *fp;
208	uint16_t nb_tx_pkts;
209	uint16_t nb_pkt_sent = 0;
210	uint32_t ret;
211
212	txq = p_txq;
213	sc = txq->sc;
214	fp = &sc->fp[txq->queue_id];
215
216	if ((unlikely((txq->nb_tx_desc - txq->nb_tx_avail) >
217				txq->tx_free_thresh)))
218		bnx2x_txeof(sc, fp);
219
220	nb_tx_pkts = RTE_MIN(nb_pkts, txq->nb_tx_avail / BDS_PER_TX_PKT);
221	if (unlikely(nb_tx_pkts == 0))
222		return 0;
223
224	while (nb_tx_pkts--) {
225		struct rte_mbuf *m = *tx_pkts++;
226		assert(m != NULL);
227		ret = bnx2x_tx_encap(txq, m);
228		fp->tx_db.data.prod += ret;
229		nb_pkt_sent++;
230	}
231
232	bnx2x_update_fp_sb_idx(fp);
233	mb();
234	DOORBELL(sc, txq->queue_id, fp->tx_db.raw);
235	mb();
236
237	if ((txq->nb_tx_desc - txq->nb_tx_avail) >
238				txq->tx_free_thresh)
239		bnx2x_txeof(sc, fp);
240
241	return nb_pkt_sent;
242}
243
244int
245bnx2x_dev_tx_queue_setup(struct rte_eth_dev *dev,
246		       uint16_t queue_idx,
247		       uint16_t nb_desc,
248		       unsigned int socket_id,
249		       const struct rte_eth_txconf *tx_conf)
250{
251	uint16_t i;
252	unsigned int tsize;
253	const struct rte_memzone *tz;
254	struct bnx2x_tx_queue *txq;
255	struct eth_tx_next_bd *tx_n_bd;
256	uint64_t busaddr;
257	struct bnx2x_softc *sc = dev->data->dev_private;
258	struct bnx2x_fastpath *fp = &sc->fp[queue_idx];
259
260	/* First allocate the tx queue data structure */
261	txq = rte_zmalloc("ethdev TX queue", sizeof(struct bnx2x_tx_queue),
262			  RTE_CACHE_LINE_SIZE);
263	if (txq == NULL)
264		return -ENOMEM;
265	txq->sc = sc;
266
267	txq->nb_tx_pages = 1;
268	while (USABLE_TX_BD(txq) < nb_desc)
269		txq->nb_tx_pages <<= 1;
270
271	txq->nb_tx_desc  = TOTAL_TX_BD(txq);
272	sc->tx_ring_size = TOTAL_TX_BD(txq);
273
274	txq->tx_free_thresh = tx_conf->tx_free_thresh ?
275		tx_conf->tx_free_thresh : DEFAULT_TX_FREE_THRESH;
276	txq->tx_free_thresh = min(txq->tx_free_thresh,
277				  txq->nb_tx_desc - BDS_PER_TX_PKT);
278
279	PMD_DRV_LOG(DEBUG, sc, "fp[%02d] req_bd=%u, thresh=%u, usable_bd=%lu, "
280		     "total_bd=%lu, tx_pages=%u",
281		     queue_idx, nb_desc, txq->tx_free_thresh,
282		     (unsigned long)USABLE_TX_BD(txq),
283		     (unsigned long)TOTAL_TX_BD(txq), txq->nb_tx_pages);
284
285	/* Allocate TX ring hardware descriptors */
286	tsize = txq->nb_tx_desc * sizeof(union eth_tx_bd_types);
287	tz = ring_dma_zone_reserve(dev, "tx_hw_ring", queue_idx, tsize, socket_id);
288	if (tz == NULL) {
289		bnx2x_tx_queue_release(txq);
290		return -ENOMEM;
291	}
292	fp->tx_desc_mapping = txq->tx_ring_phys_addr = (uint64_t)tz->phys_addr;
293	txq->tx_ring = (union eth_tx_bd_types *) tz->addr;
294	memset(txq->tx_ring, 0, tsize);
295
296	/* Allocate software ring */
297	tsize = txq->nb_tx_desc * sizeof(struct rte_mbuf *);
298	txq->sw_ring = rte_zmalloc("tx_sw_ring", tsize,
299				   RTE_CACHE_LINE_SIZE);
300	if (txq->sw_ring == NULL) {
301		bnx2x_tx_queue_release(txq);
302		return -ENOMEM;
303	}
304
305	/* PMD_DRV_LOG(DEBUG, sc, "sw_ring=%p hw_ring=%p dma_addr=0x%"PRIx64,
306	   txq->sw_ring, txq->tx_ring, txq->tx_ring_phys_addr); */
307
308	/* Link TX pages */
309	for (i = 1; i <= txq->nb_tx_pages; i++) {
310		tx_n_bd = &txq->tx_ring[TOTAL_TX_BD_PER_PAGE * i - 1].next_bd;
311		busaddr = txq->tx_ring_phys_addr + BNX2X_PAGE_SIZE * (i % txq->nb_tx_pages);
312		tx_n_bd->addr_hi = rte_cpu_to_le_32(U64_HI(busaddr));
313		tx_n_bd->addr_lo = rte_cpu_to_le_32(U64_LO(busaddr));
314		/* PMD_DRV_LOG(DEBUG, sc, "link tx page %lu",
315		 *          (TOTAL_TX_BD_PER_PAGE * i - 1));
316		 */
317	}
318
319	txq->queue_id = queue_idx;
320	txq->port_id = dev->data->port_id;
321	txq->tx_pkt_tail = 0;
322	txq->tx_pkt_head = 0;
323	txq->tx_bd_tail = 0;
324	txq->tx_bd_head = 0;
325	txq->nb_tx_avail = txq->nb_tx_desc;
326	dev->tx_pkt_burst = bnx2x_xmit_pkts;
327	dev->data->tx_queues[queue_idx] = txq;
328	if (!sc->tx_queues) sc->tx_queues = dev->data->tx_queues;
329
330	return 0;
331}
332
333static inline void
334bnx2x_upd_rx_prod_fast(struct bnx2x_softc *sc, struct bnx2x_fastpath *fp,
335		uint16_t rx_bd_prod, uint16_t rx_cq_prod)
336{
337	union ustorm_eth_rx_producers rx_prods;
338
339	rx_prods.prod.bd_prod  = rx_bd_prod;
340	rx_prods.prod.cqe_prod = rx_cq_prod;
341
342	REG_WR(sc, fp->ustorm_rx_prods_offset, rx_prods.raw_data[0]);
343}
344
345static uint16_t
346bnx2x_recv_pkts(void *p_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
347{
348	struct bnx2x_rx_queue *rxq = p_rxq;
349	struct bnx2x_softc *sc = rxq->sc;
350	struct bnx2x_fastpath *fp = &sc->fp[rxq->queue_id];
351	uint32_t nb_rx = 0;
352	uint16_t hw_cq_cons, sw_cq_cons, sw_cq_prod;
353	uint16_t bd_cons, bd_prod;
354	struct rte_mbuf *new_mb;
355	uint16_t rx_pref;
356	struct eth_fast_path_rx_cqe *cqe_fp;
357	uint16_t len, pad;
358	struct rte_mbuf *rx_mb = NULL;
359
360	hw_cq_cons = le16toh(*fp->rx_cq_cons_sb);
361	if ((hw_cq_cons & USABLE_RCQ_ENTRIES_PER_PAGE) ==
362			USABLE_RCQ_ENTRIES_PER_PAGE) {
363		++hw_cq_cons;
364	}
365
366	bd_cons = rxq->rx_bd_head;
367	bd_prod = rxq->rx_bd_tail;
368	sw_cq_cons = rxq->rx_cq_head;
369	sw_cq_prod = rxq->rx_cq_tail;
370
371	if (sw_cq_cons == hw_cq_cons)
372		return 0;
373
374	while (nb_rx < nb_pkts && sw_cq_cons != hw_cq_cons) {
375
376		bd_prod &= MAX_RX_BD(rxq);
377		bd_cons &= MAX_RX_BD(rxq);
378
379		cqe_fp = &rxq->cq_ring[sw_cq_cons & MAX_RX_BD(rxq)].fast_path_cqe;
380
381		if (unlikely(CQE_TYPE_SLOW(cqe_fp->type_error_flags & ETH_FAST_PATH_RX_CQE_TYPE))) {
382			PMD_RX_LOG(ERR, "slowpath event during traffic processing");
383			break;
384		}
385
386		if (unlikely(cqe_fp->type_error_flags & ETH_FAST_PATH_RX_CQE_PHY_DECODE_ERR_FLG)) {
387			PMD_RX_LOG(ERR, "flags 0x%x rx packet %u",
388					cqe_fp->type_error_flags, sw_cq_cons);
389			goto next_rx;
390		}
391
392		len = cqe_fp->pkt_len_or_gro_seg_len;
393		pad = cqe_fp->placement_offset;
394
395		new_mb = rte_mbuf_raw_alloc(rxq->mb_pool);
396		if (unlikely(!new_mb)) {
397			PMD_RX_LOG(ERR, "mbuf alloc fail fp[%02d]", fp->index);
398			rte_eth_devices[rxq->port_id].data->
399					rx_mbuf_alloc_failed++;
400			goto next_rx;
401		}
402
403		rx_mb = rxq->sw_ring[bd_cons];
404		rxq->sw_ring[bd_cons] = new_mb;
405		rxq->rx_ring[bd_prod] = new_mb->buf_physaddr;
406
407		rx_pref = NEXT_RX_BD(bd_cons) & MAX_RX_BD(rxq);
408		rte_prefetch0(rxq->sw_ring[rx_pref]);
409		if ((rx_pref & 0x3) == 0) {
410			rte_prefetch0(&rxq->rx_ring[rx_pref]);
411			rte_prefetch0(&rxq->sw_ring[rx_pref]);
412		}
413
414		rx_mb->data_off = pad;
415		rx_mb->nb_segs = 1;
416		rx_mb->next = NULL;
417		rx_mb->pkt_len = rx_mb->data_len = len;
418		rx_mb->port = rxq->port_id;
419		rte_prefetch1(rte_pktmbuf_mtod(rx_mb, void *));
420
421		/*
422		 * If we received a packet with a vlan tag,
423		 * attach that information to the packet.
424		 */
425		if (cqe_fp->pars_flags.flags & PARSING_FLAGS_VLAN) {
426			rx_mb->vlan_tci = cqe_fp->vlan_tag;
427			rx_mb->ol_flags |= PKT_RX_VLAN_PKT;
428		}
429
430		rx_pkts[nb_rx] = rx_mb;
431		nb_rx++;
432
433		/* limit spinning on the queue */
434		if (unlikely(nb_rx == sc->rx_budget)) {
435			PMD_RX_LOG(ERR, "Limit spinning on the queue");
436			break;
437		}
438
439next_rx:
440		bd_cons    = NEXT_RX_BD(bd_cons);
441		bd_prod    = NEXT_RX_BD(bd_prod);
442		sw_cq_prod = NEXT_RCQ_IDX(sw_cq_prod);
443		sw_cq_cons = NEXT_RCQ_IDX(sw_cq_cons);
444	}
445	rxq->rx_bd_head = bd_cons;
446	rxq->rx_bd_tail = bd_prod;
447	rxq->rx_cq_head = sw_cq_cons;
448	rxq->rx_cq_tail = sw_cq_prod;
449
450	bnx2x_upd_rx_prod_fast(sc, fp, bd_prod, sw_cq_prod);
451
452	return nb_rx;
453}
454
455int
456bnx2x_dev_rx_init(struct rte_eth_dev *dev)
457{
458	dev->rx_pkt_burst = bnx2x_recv_pkts;
459
460	return 0;
461}
462
463void
464bnx2x_dev_clear_queues(struct rte_eth_dev *dev)
465{
466	struct bnx2x_softc *sc = dev->data->dev_private;
467	uint8_t i;
468
469	PMD_INIT_FUNC_TRACE(sc);
470
471	for (i = 0; i < dev->data->nb_tx_queues; i++) {
472		struct bnx2x_tx_queue *txq = dev->data->tx_queues[i];
473		if (txq != NULL) {
474			bnx2x_tx_queue_release(txq);
475			dev->data->tx_queues[i] = NULL;
476		}
477	}
478
479	for (i = 0; i < dev->data->nb_rx_queues; i++) {
480		struct bnx2x_rx_queue *rxq = dev->data->rx_queues[i];
481		if (rxq != NULL) {
482			bnx2x_rx_queue_release(rxq);
483			dev->data->rx_queues[i] = NULL;
484		}
485	}
486}
487