mpipe_tilegx.c revision 809f0800
1/*-
2 *   BSD LICENSE
3 *
4 *   Copyright(c) 2015 EZchip Semiconductor Ltd. All rights reserved.
5 *
6 *   Redistribution and use in source and binary forms, with or without
7 *   modification, are permitted provided that the following conditions
8 *   are met:
9 *
10 *     * Redistributions of source code must retain the above copyright
11 *       notice, this list of conditions and the following disclaimer.
12 *     * Redistributions in binary form must reproduce the above copyright
13 *       notice, this list of conditions and the following disclaimer in
14 *       the documentation and/or other materials provided with the
15 *       distribution.
16 *     * Neither the name of EZchip Semiconductor nor the names of its
17 *       contributors may be used to endorse or promote products derived
18 *       from this software without specific prior written permission.
19 *
20 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31 */
32
33#include <unistd.h>
34
35#include <rte_eal.h>
36#include <rte_dev.h>
37#include <rte_eal_memconfig.h>
38#include <rte_ethdev.h>
39#include <rte_malloc.h>
40#include <rte_cycles.h>
41
42#include <arch/mpipe_xaui_def.h>
43#include <arch/mpipe_gbe_def.h>
44
45#include <gxio/mpipe.h>
46
47#ifdef RTE_LIBRTE_MPIPE_PMD_DEBUG
48#define PMD_DEBUG_RX(...)	RTE_LOG(DEBUG, PMD, __VA_ARGS__)
49#define PMD_DEBUG_TX(...)	RTE_LOG(DEBUG, PMD, __VA_ARGS__)
50#else
51#define PMD_DEBUG_RX(...)
52#define PMD_DEBUG_TX(...)
53#endif
54
55#define MPIPE_MAX_CHANNELS		128
56#define MPIPE_TX_MAX_QUEUES		128
57#define MPIPE_RX_MAX_QUEUES		16
58#define MPIPE_TX_DESCS			512
59#define MPIPE_RX_BUCKETS		256
60#define MPIPE_RX_STACK_SIZE		65536
61#define MPIPE_RX_IP_ALIGN		2
62#define MPIPE_BSM_ALIGN			128
63
64#define MPIPE_LINK_UPDATE_TIMEOUT	10	/*  s */
65#define MPIPE_LINK_UPDATE_INTERVAL	100000	/* us */
66
67struct mpipe_channel_config {
68	int enable;
69	int first_bucket;
70	int num_buckets;
71	int head_room;
72	gxio_mpipe_rules_stacks_t stacks;
73};
74
75struct mpipe_context {
76	rte_spinlock_t        lock;
77	gxio_mpipe_context_t  context;
78	struct mpipe_channel_config channels[MPIPE_MAX_CHANNELS];
79};
80
81/* Per-core local data. */
82struct mpipe_local {
83	int mbuf_push_debt[RTE_MAX_ETHPORTS];	/* Buffer push debt. */
84} __rte_cache_aligned;
85
86#define MPIPE_BUF_DEBT_THRESHOLD	32
87static __thread struct mpipe_local mpipe_local;
88static struct mpipe_context mpipe_contexts[GXIO_MPIPE_INSTANCE_MAX];
89static int mpipe_instances;
90static const char *drivername = "MPIPE PMD";
91
92/* Per queue statistics. */
93struct mpipe_queue_stats {
94	uint64_t packets, bytes, errors, nomem;
95};
96
97/* Common tx/rx queue fields. */
98struct mpipe_queue {
99	struct mpipe_dev_priv *priv;	/* "priv" data of its device. */
100	uint16_t nb_desc;		/* Number of tx descriptors. */
101	uint16_t port_id;		/* Device index. */
102	uint16_t stat_idx;		/* Queue stats index. */
103	uint8_t queue_idx;		/* Queue index. */
104	uint8_t link_status;		/* 0 = link down. */
105	struct mpipe_queue_stats stats;	/* Stat data for the queue. */
106};
107
108/* Transmit queue description. */
109struct mpipe_tx_queue {
110	struct mpipe_queue q;		/* Common stuff. */
111};
112
113/* Receive queue description. */
114struct mpipe_rx_queue {
115	struct mpipe_queue q;		/* Common stuff. */
116	gxio_mpipe_iqueue_t iqueue;	/* mPIPE iqueue. */
117	gxio_mpipe_idesc_t *next_desc;	/* Next idesc to process. */
118	int avail_descs;		/* Number of available descs. */
119	void *rx_ring_mem;		/* DMA ring memory. */
120};
121
122struct mpipe_dev_priv {
123	gxio_mpipe_context_t *context;	/* mPIPE context. */
124	gxio_mpipe_link_t link;		/* mPIPE link for the device. */
125	gxio_mpipe_equeue_t equeue;	/* mPIPE equeue. */
126	unsigned equeue_size;		/* mPIPE equeue desc count. */
127	int instance;			/* mPIPE instance. */
128	int ering;			/* mPIPE eDMA ring. */
129	int stack;			/* mPIPE buffer stack. */
130	int channel;			/* Device channel. */
131	int port_id;			/* DPDK port index. */
132	struct rte_eth_dev *eth_dev;	/* DPDK device. */
133	struct rte_mbuf **tx_comps;	/* TX completion array. */
134	struct rte_mempool *rx_mpool;	/* mpool used by the rx queues. */
135	unsigned rx_offset;		/* Receive head room. */
136	unsigned rx_size_code;		/* mPIPE rx buffer size code. */
137	int is_xaui:1,			/* Is this an xgbe or gbe? */
138	    initialized:1,		/* Initialized port? */
139	    running:1;			/* Running port? */
140	struct ether_addr mac_addr;	/* MAC address. */
141	unsigned nb_rx_queues;		/* Configured tx queues. */
142	unsigned nb_tx_queues;		/* Configured rx queues. */
143	int first_bucket;		/* mPIPE bucket start index. */
144	int first_ring;			/* mPIPE notif ring start index. */
145	int notif_group;		/* mPIPE notif group. */
146	rte_atomic32_t dp_count __rte_cache_aligned;	/* DP Entry count. */
147	int tx_stat_mapping[RTE_ETHDEV_QUEUE_STAT_CNTRS];
148	int rx_stat_mapping[RTE_ETHDEV_QUEUE_STAT_CNTRS];
149};
150
151#define mpipe_priv(dev)			\
152	((struct mpipe_dev_priv*)(dev)->data->dev_private)
153
154#define mpipe_name(priv)		\
155	((priv)->eth_dev->data->name)
156
157#define mpipe_rx_queue(priv, n)		\
158	((struct mpipe_rx_queue *)(priv)->eth_dev->data->rx_queues[n])
159
160#define mpipe_tx_queue(priv, n)		\
161	((struct mpipe_tx_queue *)(priv)->eth_dev->data->tx_queues[n])
162
163static void
164mpipe_xmit_flush(struct mpipe_dev_priv *priv);
165
166static void
167mpipe_recv_flush(struct mpipe_dev_priv *priv);
168
169static int mpipe_equeue_sizes[] = {
170	[GXIO_MPIPE_EQUEUE_ENTRY_512]	= 512,
171	[GXIO_MPIPE_EQUEUE_ENTRY_2K]	= 2048,
172	[GXIO_MPIPE_EQUEUE_ENTRY_8K]	= 8192,
173	[GXIO_MPIPE_EQUEUE_ENTRY_64K]	= 65536,
174};
175
176static int mpipe_iqueue_sizes[] = {
177	[GXIO_MPIPE_IQUEUE_ENTRY_128]	= 128,
178	[GXIO_MPIPE_IQUEUE_ENTRY_512]	= 512,
179	[GXIO_MPIPE_IQUEUE_ENTRY_2K]	= 2048,
180	[GXIO_MPIPE_IQUEUE_ENTRY_64K]	= 65536,
181};
182
183static int mpipe_buffer_sizes[] = {
184	[GXIO_MPIPE_BUFFER_SIZE_128]	= 128,
185	[GXIO_MPIPE_BUFFER_SIZE_256]	= 256,
186	[GXIO_MPIPE_BUFFER_SIZE_512]	= 512,
187	[GXIO_MPIPE_BUFFER_SIZE_1024]	= 1024,
188	[GXIO_MPIPE_BUFFER_SIZE_1664]	= 1664,
189	[GXIO_MPIPE_BUFFER_SIZE_4096]	= 4096,
190	[GXIO_MPIPE_BUFFER_SIZE_10368]	= 10368,
191	[GXIO_MPIPE_BUFFER_SIZE_16384]	= 16384,
192};
193
194static gxio_mpipe_context_t *
195mpipe_context(int instance)
196{
197	if (instance < 0 || instance >= mpipe_instances)
198		return NULL;
199	return &mpipe_contexts[instance].context;
200}
201
202static int mpipe_channel_config(int instance, int channel,
203				struct mpipe_channel_config *config)
204{
205	struct mpipe_channel_config *data;
206	struct mpipe_context *context;
207	gxio_mpipe_rules_t rules;
208	int idx, rc = 0;
209
210	if (instance < 0 || instance >= mpipe_instances ||
211	    channel < 0 || channel >= MPIPE_MAX_CHANNELS)
212		return -EINVAL;
213
214	context = &mpipe_contexts[instance];
215
216	rte_spinlock_lock(&context->lock);
217
218	gxio_mpipe_rules_init(&rules, &context->context);
219
220	for (idx = 0; idx < MPIPE_MAX_CHANNELS; idx++) {
221		data = (channel == idx) ? config : &context->channels[idx];
222
223		if (!data->enable)
224			continue;
225
226		rc = gxio_mpipe_rules_begin(&rules, data->first_bucket,
227					    data->num_buckets, &data->stacks);
228		if (rc < 0) {
229			goto done;
230		}
231
232		rc = gxio_mpipe_rules_add_channel(&rules, idx);
233		if (rc < 0) {
234			goto done;
235		}
236
237		rc = gxio_mpipe_rules_set_headroom(&rules, data->head_room);
238		if (rc < 0) {
239			goto done;
240		}
241	}
242
243	rc = gxio_mpipe_rules_commit(&rules);
244	if (rc == 0) {
245		memcpy(&context->channels[channel], config, sizeof(*config));
246	}
247
248done:
249	rte_spinlock_unlock(&context->lock);
250
251	return rc;
252}
253
254static int
255mpipe_get_size_index(int *array, int count, int size,
256		     bool roundup)
257{
258	int i, last = -1;
259
260	for (i = 0; i < count && array[i] < size; i++) {
261		if (array[i])
262			last = i;
263	}
264
265	if (roundup)
266		return i < count ? (int)i : -ENOENT;
267	else
268		return last >= 0 ? last : -ENOENT;
269}
270
271static int
272mpipe_calc_size(int *array, int count, int size)
273{
274	int index = mpipe_get_size_index(array, count, size, 1);
275	return index < 0 ? index : array[index];
276}
277
278static int mpipe_equeue_size(int size)
279{
280	int result;
281	result = mpipe_calc_size(mpipe_equeue_sizes,
282				 RTE_DIM(mpipe_equeue_sizes), size);
283	return result;
284}
285
286static int mpipe_iqueue_size(int size)
287{
288	int result;
289	result = mpipe_calc_size(mpipe_iqueue_sizes,
290				 RTE_DIM(mpipe_iqueue_sizes), size);
291	return result;
292}
293
294static int mpipe_buffer_size_index(int size)
295{
296	int result;
297	result = mpipe_get_size_index(mpipe_buffer_sizes,
298				      RTE_DIM(mpipe_buffer_sizes), size, 0);
299	return result;
300}
301
302static inline int
303mpipe_dev_atomic_read_link_status(struct rte_eth_dev *dev,
304				  struct rte_eth_link *link)
305{
306	struct rte_eth_link *dst = link;
307	struct rte_eth_link *src = &(dev->data->dev_link);
308
309	if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
310				*(uint64_t *)src) == 0)
311		return -1;
312
313	return 0;
314}
315
316static inline int
317mpipe_dev_atomic_write_link_status(struct rte_eth_dev *dev,
318				   struct rte_eth_link *link)
319{
320	struct rte_eth_link *dst = &(dev->data->dev_link);
321	struct rte_eth_link *src = link;
322
323	if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
324				*(uint64_t *)src) == 0)
325		return -1;
326
327	return 0;
328}
329
330static void
331mpipe_infos_get(struct rte_eth_dev *dev __rte_unused,
332		struct rte_eth_dev_info *dev_info)
333{
334	dev_info->min_rx_bufsize  = 128;
335	dev_info->max_rx_pktlen   = 1518;
336	dev_info->max_tx_queues   = MPIPE_TX_MAX_QUEUES;
337	dev_info->max_rx_queues   = MPIPE_RX_MAX_QUEUES;
338	dev_info->max_mac_addrs   = 1;
339	dev_info->rx_offload_capa = 0;
340	dev_info->tx_offload_capa = 0;
341}
342
343static int
344mpipe_configure(struct rte_eth_dev *dev)
345{
346	struct mpipe_dev_priv *priv = mpipe_priv(dev);
347
348	if (dev->data->nb_tx_queues > MPIPE_TX_MAX_QUEUES) {
349		RTE_LOG(ERR, PMD, "%s: Too many tx queues: %d > %d\n",
350			mpipe_name(priv), dev->data->nb_tx_queues,
351			MPIPE_TX_MAX_QUEUES);
352		return -EINVAL;
353	}
354	priv->nb_tx_queues = dev->data->nb_tx_queues;
355
356	if (dev->data->nb_rx_queues > MPIPE_RX_MAX_QUEUES) {
357		RTE_LOG(ERR, PMD, "%s: Too many rx queues: %d > %d\n",
358			mpipe_name(priv), dev->data->nb_rx_queues,
359			MPIPE_RX_MAX_QUEUES);
360	}
361	priv->nb_rx_queues = dev->data->nb_rx_queues;
362
363	return 0;
364}
365
366static inline int
367mpipe_link_compare(struct rte_eth_link *link1,
368		   struct rte_eth_link *link2)
369{
370	return (*(uint64_t *)link1 == *(uint64_t *)link2)
371		? -1 : 0;
372}
373
374static int
375mpipe_link_update(struct rte_eth_dev *dev, int wait_to_complete)
376{
377	struct mpipe_dev_priv *priv = mpipe_priv(dev);
378	struct rte_eth_link old, new;
379	int64_t state, speed;
380	int count, rc;
381
382	memset(&old, 0, sizeof(old));
383	memset(&new, 0, sizeof(new));
384	mpipe_dev_atomic_read_link_status(dev, &old);
385
386	for (count = 0, rc = 0; count < MPIPE_LINK_UPDATE_TIMEOUT; count++) {
387		if (!priv->initialized)
388			break;
389
390		state = gxio_mpipe_link_get_attr(&priv->link,
391						 GXIO_MPIPE_LINK_CURRENT_STATE);
392		if (state < 0)
393			break;
394
395		speed = state & GXIO_MPIPE_LINK_SPEED_MASK;
396
397		new.link_autoneg = (dev->data->dev_conf.link_speeds &
398				ETH_LINK_SPEED_AUTONEG);
399		if (speed == GXIO_MPIPE_LINK_1G) {
400			new.link_speed = ETH_SPEED_NUM_1G;
401			new.link_duplex = ETH_LINK_FULL_DUPLEX;
402			new.link_status = ETH_LINK_UP;
403		} else if (speed == GXIO_MPIPE_LINK_10G) {
404			new.link_speed = ETH_SPEED_NUM_10G;
405			new.link_duplex = ETH_LINK_FULL_DUPLEX;
406			new.link_status = ETH_LINK_UP;
407		}
408
409		rc = mpipe_link_compare(&old, &new);
410		if (rc == 0 || !wait_to_complete)
411			break;
412
413		rte_delay_us(MPIPE_LINK_UPDATE_INTERVAL);
414	}
415
416	mpipe_dev_atomic_write_link_status(dev, &new);
417	return rc;
418}
419
420static int
421mpipe_set_link(struct rte_eth_dev *dev, int up)
422{
423	struct mpipe_dev_priv *priv = mpipe_priv(dev);
424	int rc;
425
426	rc = gxio_mpipe_link_set_attr(&priv->link,
427				      GXIO_MPIPE_LINK_DESIRED_STATE,
428				      up ? GXIO_MPIPE_LINK_ANYSPEED : 0);
429	if (rc < 0) {
430		RTE_LOG(ERR, PMD, "%s: Failed to set link %s.\n",
431			mpipe_name(priv), up ? "up" : "down");
432	} else {
433		mpipe_link_update(dev, 0);
434	}
435
436	return rc;
437}
438
439static int
440mpipe_set_link_up(struct rte_eth_dev *dev)
441{
442	return mpipe_set_link(dev, 1);
443}
444
445static int
446mpipe_set_link_down(struct rte_eth_dev *dev)
447{
448	return mpipe_set_link(dev, 0);
449}
450
451static inline void
452mpipe_dp_enter(struct mpipe_dev_priv *priv)
453{
454	__insn_mtspr(SPR_DSTREAM_PF, 0);
455	rte_atomic32_inc(&priv->dp_count);
456}
457
458static inline void
459mpipe_dp_exit(struct mpipe_dev_priv *priv)
460{
461	rte_atomic32_dec(&priv->dp_count);
462}
463
464static inline void
465mpipe_dp_wait(struct mpipe_dev_priv *priv)
466{
467	while (rte_atomic32_read(&priv->dp_count) != 0) {
468		rte_pause();
469	}
470}
471
472static inline int
473mpipe_mbuf_stack_index(struct mpipe_dev_priv *priv, struct rte_mbuf *mbuf)
474{
475	return (mbuf->port < RTE_MAX_ETHPORTS) ?
476		mpipe_priv(&rte_eth_devices[mbuf->port])->stack :
477		priv->stack;
478}
479
480static inline struct rte_mbuf *
481mpipe_recv_mbuf(struct mpipe_dev_priv *priv, gxio_mpipe_idesc_t *idesc,
482		int in_port)
483{
484	void *va = gxio_mpipe_idesc_get_va(idesc);
485	uint16_t size = gxio_mpipe_idesc_get_xfer_size(idesc);
486	struct rte_mbuf *mbuf = RTE_PTR_SUB(va, priv->rx_offset);
487
488	rte_pktmbuf_reset(mbuf);
489	mbuf->data_off = (uintptr_t)va - (uintptr_t)mbuf->buf_addr;
490	mbuf->port     = in_port;
491	mbuf->data_len = size;
492	mbuf->pkt_len  = size;
493	mbuf->hash.rss = gxio_mpipe_idesc_get_flow_hash(idesc);
494
495	PMD_DEBUG_RX("%s: RX mbuf %p, buffer %p, buf_addr %p, size %d\n",
496		     mpipe_name(priv), mbuf, va, mbuf->buf_addr, size);
497
498	return mbuf;
499}
500
501static inline void
502mpipe_recv_push(struct mpipe_dev_priv *priv, struct rte_mbuf *mbuf)
503{
504	const int offset = RTE_PKTMBUF_HEADROOM + MPIPE_RX_IP_ALIGN;
505	void *buf_addr = RTE_PTR_ADD(mbuf->buf_addr, offset);
506
507	gxio_mpipe_push_buffer(priv->context, priv->stack, buf_addr);
508	PMD_DEBUG_RX("%s: Pushed mbuf %p, buffer %p into stack %d\n",
509		     mpipe_name(priv), mbuf, buf_addr, priv->stack);
510}
511
512static inline void
513mpipe_recv_fill_stack(struct mpipe_dev_priv *priv, int count)
514{
515	struct rte_mbuf *mbuf;
516	int i;
517
518	for (i = 0; i < count; i++) {
519		mbuf = rte_mbuf_raw_alloc(priv->rx_mpool);
520		if (!mbuf)
521			break;
522		mpipe_recv_push(priv, mbuf);
523	}
524
525	PMD_DEBUG_RX("%s: Filled %d/%d buffers\n", mpipe_name(priv), i, count);
526}
527
528static inline void
529mpipe_recv_flush_stack(struct mpipe_dev_priv *priv)
530{
531	const int offset = priv->rx_offset & ~RTE_MEMPOOL_ALIGN_MASK;
532	uint8_t in_port = priv->port_id;
533	struct rte_mbuf *mbuf;
534	void *va;
535
536	while (1) {
537		va = gxio_mpipe_pop_buffer(priv->context, priv->stack);
538		if (!va)
539			break;
540		mbuf = RTE_PTR_SUB(va, offset);
541
542		PMD_DEBUG_RX("%s: Flushing mbuf %p, va %p\n",
543			     mpipe_name(priv), mbuf, va);
544
545		mbuf->data_off    = (uintptr_t)va - (uintptr_t)mbuf->buf_addr;
546		mbuf->refcnt      = 1;
547		mbuf->nb_segs     = 1;
548		mbuf->port        = in_port;
549		mbuf->packet_type = 0;
550		mbuf->data_len    = 0;
551		mbuf->pkt_len     = 0;
552
553		__rte_mbuf_raw_free(mbuf);
554	}
555}
556
557static void
558mpipe_register_segment(struct mpipe_dev_priv *priv, const struct rte_memseg *ms)
559{
560	size_t size = ms->hugepage_sz;
561	uint8_t *addr, *end;
562	int rc;
563
564	for (addr = ms->addr, end = addr + ms->len; addr < end; addr += size) {
565		rc = gxio_mpipe_register_page(priv->context, priv->stack, addr,
566					      size, 0);
567		if (rc < 0)
568			break;
569	}
570
571	if (rc < 0) {
572		RTE_LOG(ERR, PMD, "%s: Could not register memseg @%p, %d.\n",
573			mpipe_name(priv), ms->addr, rc);
574	} else {
575		RTE_LOG(DEBUG, PMD, "%s: Registered segment %p - %p\n",
576			mpipe_name(priv), ms->addr,
577			RTE_PTR_ADD(ms->addr, ms->len - 1));
578	}
579}
580
581static int
582mpipe_recv_init(struct mpipe_dev_priv *priv)
583{
584	const struct rte_memseg *seg = rte_eal_get_physmem_layout();
585	size_t stack_size;
586	void *stack_mem;
587	int rc;
588
589	if (!priv->rx_mpool) {
590		RTE_LOG(ERR, PMD, "%s: No buffer pool.\n",
591			mpipe_name(priv));
592		return -ENODEV;
593	}
594
595	/* Allocate one NotifRing for each queue. */
596	rc = gxio_mpipe_alloc_notif_rings(priv->context, MPIPE_RX_MAX_QUEUES,
597					  0, 0);
598	if (rc < 0) {
599		RTE_LOG(ERR, PMD, "%s: Failed to allocate notif rings.\n",
600			mpipe_name(priv));
601		return rc;
602	}
603	priv->first_ring = rc;
604
605	/* Allocate a NotifGroup. */
606	rc = gxio_mpipe_alloc_notif_groups(priv->context, 1, 0, 0);
607	if (rc < 0) {
608		RTE_LOG(ERR, PMD, "%s: Failed to allocate rx group.\n",
609			mpipe_name(priv));
610		return rc;
611	}
612	priv->notif_group = rc;
613
614	/* Allocate required buckets. */
615	rc = gxio_mpipe_alloc_buckets(priv->context, MPIPE_RX_BUCKETS, 0, 0);
616	if (rc < 0) {
617		RTE_LOG(ERR, PMD, "%s: Failed to allocate buckets.\n",
618			mpipe_name(priv));
619		return rc;
620	}
621	priv->first_bucket = rc;
622
623	rc = gxio_mpipe_alloc_buffer_stacks(priv->context, 1, 0, 0);
624	if (rc < 0) {
625		RTE_LOG(ERR, PMD, "%s: Failed to allocate buffer stack.\n",
626			mpipe_name(priv));
627		return rc;
628	}
629	priv->stack = rc;
630
631	while (seg && seg->addr)
632		mpipe_register_segment(priv, seg++);
633
634	stack_size = gxio_mpipe_calc_buffer_stack_bytes(MPIPE_RX_STACK_SIZE);
635	stack_mem = rte_zmalloc(NULL, stack_size, 65536);
636	if (!stack_mem) {
637		RTE_LOG(ERR, PMD, "%s: Failed to allocate buffer memory.\n",
638			mpipe_name(priv));
639		return -ENOMEM;
640	} else {
641		RTE_LOG(DEBUG, PMD, "%s: Buffer stack memory %p - %p.\n",
642			mpipe_name(priv), stack_mem,
643			RTE_PTR_ADD(stack_mem, stack_size - 1));
644	}
645
646	rc = gxio_mpipe_init_buffer_stack(priv->context, priv->stack,
647					  priv->rx_size_code, stack_mem,
648					  stack_size, 0);
649	if (rc < 0) {
650		RTE_LOG(ERR, PMD, "%s: Failed to initialize buffer stack.\n",
651			mpipe_name(priv));
652		return rc;
653	}
654
655	return 0;
656}
657
658static int
659mpipe_xmit_init(struct mpipe_dev_priv *priv)
660{
661	size_t ring_size;
662	void *ring_mem;
663	int rc;
664
665	/* Allocate eDMA ring. */
666	rc = gxio_mpipe_alloc_edma_rings(priv->context, 1, 0, 0);
667	if (rc < 0) {
668		RTE_LOG(ERR, PMD, "%s: Failed to alloc tx ring.\n",
669			mpipe_name(priv));
670		return rc;
671	}
672	priv->ering = rc;
673
674	rc = mpipe_equeue_size(MPIPE_TX_DESCS);
675	if (rc < 0) {
676		RTE_LOG(ERR, PMD, "%s: Cannot allocate %d equeue descs.\n",
677			mpipe_name(priv), (int)MPIPE_TX_DESCS);
678		return -ENOMEM;
679	}
680	priv->equeue_size = rc;
681
682	/* Initialize completion array. */
683	ring_size = sizeof(priv->tx_comps[0]) * priv->equeue_size;
684	priv->tx_comps = rte_zmalloc(NULL, ring_size, RTE_CACHE_LINE_SIZE);
685	if (!priv->tx_comps) {
686		RTE_LOG(ERR, PMD, "%s: Failed to allocate egress comps.\n",
687			mpipe_name(priv));
688		return -ENOMEM;
689	}
690
691	/* Allocate eDMA ring memory. */
692	ring_size = sizeof(gxio_mpipe_edesc_t) * priv->equeue_size;
693	ring_mem = rte_zmalloc(NULL, ring_size, ring_size);
694	if (!ring_mem) {
695		RTE_LOG(ERR, PMD, "%s: Failed to allocate egress descs.\n",
696			mpipe_name(priv));
697		return -ENOMEM;
698	} else {
699		RTE_LOG(DEBUG, PMD, "%s: eDMA ring memory %p - %p.\n",
700			mpipe_name(priv), ring_mem,
701			RTE_PTR_ADD(ring_mem, ring_size - 1));
702	}
703
704	/* Initialize eDMA ring. */
705	rc = gxio_mpipe_equeue_init(&priv->equeue, priv->context, priv->ering,
706				    priv->channel, ring_mem, ring_size, 0);
707	if (rc < 0) {
708		RTE_LOG(ERR, PMD, "%s: Failed to init equeue\n",
709			mpipe_name(priv));
710		return rc;
711	}
712
713	return 0;
714}
715
716static int
717mpipe_link_init(struct mpipe_dev_priv *priv)
718{
719	int rc;
720
721	/* Open the link. */
722	rc = gxio_mpipe_link_open(&priv->link, priv->context,
723				  mpipe_name(priv), GXIO_MPIPE_LINK_AUTO_NONE);
724	if (rc < 0) {
725		RTE_LOG(ERR, PMD, "%s: Failed to open link.\n",
726			mpipe_name(priv));
727		return rc;
728	}
729
730	/* Get the channel index. */
731	rc = gxio_mpipe_link_channel(&priv->link);
732	if (rc < 0) {
733		RTE_LOG(ERR, PMD, "%s: Bad channel\n",
734			mpipe_name(priv));
735		return rc;
736	}
737	priv->channel = rc;
738
739	return 0;
740}
741
742static int
743mpipe_init(struct mpipe_dev_priv *priv)
744{
745	int rc;
746
747	if (priv->initialized)
748		return 0;
749
750	rc = mpipe_recv_init(priv);
751	if (rc < 0) {
752		RTE_LOG(ERR, PMD, "%s: Failed to init rx.\n",
753			mpipe_name(priv));
754		return rc;
755	}
756
757	rc = mpipe_xmit_init(priv);
758	if (rc < 0) {
759		RTE_LOG(ERR, PMD, "%s: Failed to init tx.\n",
760			mpipe_name(priv));
761		rte_free(priv);
762		return rc;
763	}
764
765	priv->initialized = 1;
766
767	return 0;
768}
769
770static int
771mpipe_start(struct rte_eth_dev *dev)
772{
773	struct mpipe_dev_priv *priv = mpipe_priv(dev);
774	struct mpipe_channel_config config;
775	struct mpipe_rx_queue *rx_queue;
776	struct rte_eth_link eth_link;
777	unsigned queue, buffers = 0;
778	size_t ring_size;
779	void *ring_mem;
780	int rc;
781
782	memset(&eth_link, 0, sizeof(eth_link));
783	mpipe_dev_atomic_write_link_status(dev, &eth_link);
784
785	rc = mpipe_init(priv);
786	if (rc < 0)
787		return rc;
788
789	/* Initialize NotifRings. */
790	for (queue = 0; queue < priv->nb_rx_queues; queue++) {
791		rx_queue = mpipe_rx_queue(priv, queue);
792		ring_size = rx_queue->q.nb_desc * sizeof(gxio_mpipe_idesc_t);
793
794		ring_mem = rte_malloc(NULL, ring_size, ring_size);
795		if (!ring_mem) {
796			RTE_LOG(ERR, PMD, "%s: Failed to alloc rx descs.\n",
797				mpipe_name(priv));
798			return -ENOMEM;
799		} else {
800			RTE_LOG(DEBUG, PMD, "%s: iDMA ring %d memory %p - %p.\n",
801				mpipe_name(priv), queue, ring_mem,
802				RTE_PTR_ADD(ring_mem, ring_size - 1));
803		}
804
805		rc = gxio_mpipe_iqueue_init(&rx_queue->iqueue, priv->context,
806					    priv->first_ring + queue, ring_mem,
807					    ring_size, 0);
808		if (rc < 0) {
809			RTE_LOG(ERR, PMD, "%s: Failed to init rx queue.\n",
810				mpipe_name(priv));
811			return rc;
812		}
813
814		rx_queue->rx_ring_mem = ring_mem;
815		buffers += rx_queue->q.nb_desc;
816	}
817
818	/* Initialize ingress NotifGroup and buckets. */
819	rc = gxio_mpipe_init_notif_group_and_buckets(priv->context,
820			priv->notif_group, priv->first_ring, priv->nb_rx_queues,
821			priv->first_bucket, MPIPE_RX_BUCKETS,
822			GXIO_MPIPE_BUCKET_STATIC_FLOW_AFFINITY);
823	if (rc < 0) {
824		RTE_LOG(ERR, PMD, "%s: Failed to init group and buckets.\n",
825			mpipe_name(priv));
826		return rc;
827	}
828
829	/* Configure the classifier to deliver packets from this port. */
830	config.enable = 1;
831	config.first_bucket = priv->first_bucket;
832	config.num_buckets = MPIPE_RX_BUCKETS;
833	memset(&config.stacks, 0xff, sizeof(config.stacks));
834	config.stacks.stacks[priv->rx_size_code] = priv->stack;
835	config.head_room = priv->rx_offset & RTE_MEMPOOL_ALIGN_MASK;
836
837	rc = mpipe_channel_config(priv->instance, priv->channel,
838				  &config);
839	if (rc < 0) {
840		RTE_LOG(ERR, PMD, "%s: Failed to setup classifier.\n",
841			mpipe_name(priv));
842		return rc;
843	}
844
845	/* Fill empty buffers into the buffer stack. */
846	mpipe_recv_fill_stack(priv, buffers);
847
848	/* Bring up the link. */
849	mpipe_set_link_up(dev);
850
851	/* Start xmit/recv on queues. */
852	for (queue = 0; queue < priv->nb_tx_queues; queue++)
853		mpipe_tx_queue(priv, queue)->q.link_status = ETH_LINK_UP;
854	for (queue = 0; queue < priv->nb_rx_queues; queue++)
855		mpipe_rx_queue(priv, queue)->q.link_status = ETH_LINK_UP;
856	priv->running = 1;
857
858	return 0;
859}
860
861static void
862mpipe_stop(struct rte_eth_dev *dev)
863{
864	struct mpipe_dev_priv *priv = mpipe_priv(dev);
865	struct mpipe_channel_config config;
866	unsigned queue;
867	int rc;
868
869	for (queue = 0; queue < priv->nb_tx_queues; queue++)
870		mpipe_tx_queue(priv, queue)->q.link_status = ETH_LINK_DOWN;
871	for (queue = 0; queue < priv->nb_rx_queues; queue++)
872		mpipe_rx_queue(priv, queue)->q.link_status = ETH_LINK_DOWN;
873
874	/* Make sure the link_status writes land. */
875	rte_wmb();
876
877	/*
878	 * Wait for link_status change to register with straggling datapath
879	 * threads.
880	 */
881	mpipe_dp_wait(priv);
882
883	/* Bring down the link. */
884	mpipe_set_link_down(dev);
885
886	/* Remove classifier rules. */
887	memset(&config, 0, sizeof(config));
888	rc = mpipe_channel_config(priv->instance, priv->channel,
889				  &config);
890	if (rc < 0) {
891		RTE_LOG(ERR, PMD, "%s: Failed to stop classifier.\n",
892			mpipe_name(priv));
893	}
894
895	/* Flush completed xmit packets. */
896	mpipe_xmit_flush(priv);
897
898	/* Flush buffer stacks. */
899	mpipe_recv_flush(priv);
900
901	priv->running = 0;
902}
903
904static void
905mpipe_close(struct rte_eth_dev *dev)
906{
907	struct mpipe_dev_priv *priv = mpipe_priv(dev);
908	if (priv->running)
909		mpipe_stop(dev);
910}
911
912static void
913mpipe_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
914{
915	struct mpipe_dev_priv *priv = mpipe_priv(dev);
916	struct mpipe_tx_queue *tx_queue;
917	struct mpipe_rx_queue *rx_queue;
918	unsigned i;
919	uint16_t idx;
920
921	memset(stats, 0, sizeof(*stats));
922
923	for (i = 0; i < priv->nb_tx_queues; i++) {
924		tx_queue = mpipe_tx_queue(priv, i);
925
926		stats->opackets += tx_queue->q.stats.packets;
927		stats->obytes   += tx_queue->q.stats.bytes;
928		stats->oerrors  += tx_queue->q.stats.errors;
929
930		idx = tx_queue->q.stat_idx;
931		if (idx != (uint16_t)-1) {
932			stats->q_opackets[idx] += tx_queue->q.stats.packets;
933			stats->q_obytes[idx]   += tx_queue->q.stats.bytes;
934			stats->q_errors[idx]   += tx_queue->q.stats.errors;
935		}
936	}
937
938	for (i = 0; i < priv->nb_rx_queues; i++) {
939		rx_queue = mpipe_rx_queue(priv, i);
940
941		stats->ipackets  += rx_queue->q.stats.packets;
942		stats->ibytes    += rx_queue->q.stats.bytes;
943		stats->ierrors   += rx_queue->q.stats.errors;
944		stats->rx_nombuf += rx_queue->q.stats.nomem;
945
946		idx = rx_queue->q.stat_idx;
947		if (idx != (uint16_t)-1) {
948			stats->q_ipackets[idx] += rx_queue->q.stats.packets;
949			stats->q_ibytes[idx]   += rx_queue->q.stats.bytes;
950			stats->q_errors[idx]   += rx_queue->q.stats.errors;
951		}
952	}
953}
954
955static void
956mpipe_stats_reset(struct rte_eth_dev *dev)
957{
958	struct mpipe_dev_priv *priv = mpipe_priv(dev);
959	struct mpipe_tx_queue *tx_queue;
960	struct mpipe_rx_queue *rx_queue;
961	unsigned i;
962
963	for (i = 0; i < priv->nb_tx_queues; i++) {
964		tx_queue = mpipe_tx_queue(priv, i);
965		memset(&tx_queue->q.stats, 0, sizeof(tx_queue->q.stats));
966	}
967
968	for (i = 0; i < priv->nb_rx_queues; i++) {
969		rx_queue = mpipe_rx_queue(priv, i);
970		memset(&rx_queue->q.stats, 0, sizeof(rx_queue->q.stats));
971	}
972}
973
974static int
975mpipe_queue_stats_mapping_set(struct rte_eth_dev *dev, uint16_t queue_id,
976			      uint8_t stat_idx, uint8_t is_rx)
977{
978	struct mpipe_dev_priv *priv = mpipe_priv(dev);
979
980	if (is_rx) {
981		priv->rx_stat_mapping[stat_idx] = queue_id;
982	} else {
983		priv->tx_stat_mapping[stat_idx] = queue_id;
984	}
985
986	return 0;
987}
988
989static int
990mpipe_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
991		     uint16_t nb_desc, unsigned int socket_id __rte_unused,
992		     const struct rte_eth_txconf *tx_conf __rte_unused)
993{
994	struct mpipe_tx_queue *tx_queue = dev->data->tx_queues[queue_idx];
995	struct mpipe_dev_priv *priv = mpipe_priv(dev);
996	uint16_t idx;
997
998	tx_queue = rte_realloc(tx_queue, sizeof(*tx_queue),
999			       RTE_CACHE_LINE_SIZE);
1000	if (!tx_queue) {
1001		RTE_LOG(ERR, PMD, "%s: Failed to allocate TX queue.\n",
1002			mpipe_name(priv));
1003		return -ENOMEM;
1004	}
1005
1006	memset(&tx_queue->q, 0, sizeof(tx_queue->q));
1007	tx_queue->q.priv = priv;
1008	tx_queue->q.queue_idx = queue_idx;
1009	tx_queue->q.port_id = dev->data->port_id;
1010	tx_queue->q.nb_desc = nb_desc;
1011
1012	tx_queue->q.stat_idx = -1;
1013	for (idx = 0; idx < RTE_ETHDEV_QUEUE_STAT_CNTRS; idx++) {
1014		if (priv->tx_stat_mapping[idx] == queue_idx)
1015			tx_queue->q.stat_idx = idx;
1016	}
1017
1018	dev->data->tx_queues[queue_idx] = tx_queue;
1019
1020	return 0;
1021}
1022
1023static void
1024mpipe_tx_queue_release(void *_txq)
1025{
1026	rte_free(_txq);
1027}
1028
1029static int
1030mpipe_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
1031		     uint16_t nb_desc, unsigned int socket_id __rte_unused,
1032		     const struct rte_eth_rxconf *rx_conf __rte_unused,
1033		     struct rte_mempool *mp)
1034{
1035	struct mpipe_rx_queue *rx_queue = dev->data->rx_queues[queue_idx];
1036	struct mpipe_dev_priv *priv = mpipe_priv(dev);
1037	uint16_t idx;
1038	int size, rc;
1039
1040	rc = mpipe_iqueue_size(nb_desc);
1041	if (rc < 0) {
1042		RTE_LOG(ERR, PMD, "%s: Cannot allocate %d iqueue descs.\n",
1043			mpipe_name(priv), (int)nb_desc);
1044		return -ENOMEM;
1045	}
1046
1047	if (rc != nb_desc) {
1048		RTE_LOG(WARNING, PMD, "%s: Extending RX descs from %d to %d.\n",
1049			mpipe_name(priv), (int)nb_desc, rc);
1050		nb_desc = rc;
1051	}
1052
1053	size = sizeof(*rx_queue);
1054	rx_queue = rte_realloc(rx_queue, size, RTE_CACHE_LINE_SIZE);
1055	if (!rx_queue) {
1056		RTE_LOG(ERR, PMD, "%s: Failed to allocate RX queue.\n",
1057			mpipe_name(priv));
1058		return -ENOMEM;
1059	}
1060
1061	memset(&rx_queue->q, 0, sizeof(rx_queue->q));
1062	rx_queue->q.priv = priv;
1063	rx_queue->q.nb_desc = nb_desc;
1064	rx_queue->q.port_id = dev->data->port_id;
1065	rx_queue->q.queue_idx = queue_idx;
1066
1067	if (!priv->rx_mpool) {
1068		int size = (rte_pktmbuf_data_room_size(mp) -
1069			    RTE_PKTMBUF_HEADROOM -
1070			    MPIPE_RX_IP_ALIGN);
1071
1072		priv->rx_offset = (sizeof(struct rte_mbuf) +
1073				   rte_pktmbuf_priv_size(mp) +
1074				   RTE_PKTMBUF_HEADROOM +
1075				   MPIPE_RX_IP_ALIGN);
1076		if (size < 0) {
1077			RTE_LOG(ERR, PMD, "%s: Bad buffer size %d.\n",
1078				mpipe_name(priv),
1079				rte_pktmbuf_data_room_size(mp));
1080			return -ENOMEM;
1081		}
1082
1083		priv->rx_size_code = mpipe_buffer_size_index(size);
1084		priv->rx_mpool = mp;
1085	}
1086
1087	if (priv->rx_mpool != mp) {
1088		RTE_LOG(WARNING, PMD, "%s: Ignoring multiple buffer pools.\n",
1089			mpipe_name(priv));
1090	}
1091
1092	rx_queue->q.stat_idx = -1;
1093	for (idx = 0; idx < RTE_ETHDEV_QUEUE_STAT_CNTRS; idx++) {
1094		if (priv->rx_stat_mapping[idx] == queue_idx)
1095			rx_queue->q.stat_idx = idx;
1096	}
1097
1098	dev->data->rx_queues[queue_idx] = rx_queue;
1099
1100	return 0;
1101}
1102
1103static void
1104mpipe_rx_queue_release(void *_rxq)
1105{
1106	rte_free(_rxq);
1107}
1108
1109#define MPIPE_XGBE_ENA_HASH_MULTI	\
1110	(1UL << MPIPE_XAUI_RECEIVE_CONFIGURATION__ENA_HASH_MULTI_SHIFT)
1111#define MPIPE_XGBE_ENA_HASH_UNI		\
1112	(1UL << MPIPE_XAUI_RECEIVE_CONFIGURATION__ENA_HASH_UNI_SHIFT)
1113#define MPIPE_XGBE_COPY_ALL		\
1114	(1UL << MPIPE_XAUI_RECEIVE_CONFIGURATION__COPY_ALL_SHIFT)
1115#define MPIPE_GBE_ENA_MULTI_HASH	\
1116	(1UL << MPIPE_GBE_NETWORK_CONFIGURATION__MULTI_HASH_ENA_SHIFT)
1117#define MPIPE_GBE_ENA_UNI_HASH		\
1118	(1UL << MPIPE_GBE_NETWORK_CONFIGURATION__UNI_HASH_ENA_SHIFT)
1119#define MPIPE_GBE_COPY_ALL		\
1120	(1UL << MPIPE_GBE_NETWORK_CONFIGURATION__COPY_ALL_SHIFT)
1121
1122static void
1123mpipe_promiscuous_enable(struct rte_eth_dev *dev)
1124{
1125	struct mpipe_dev_priv *priv = mpipe_priv(dev);
1126	int64_t reg;
1127	int addr;
1128
1129	if (priv->is_xaui) {
1130		addr = MPIPE_XAUI_RECEIVE_CONFIGURATION;
1131		reg  = gxio_mpipe_link_mac_rd(&priv->link, addr);
1132		reg &= ~MPIPE_XGBE_ENA_HASH_MULTI;
1133		reg &= ~MPIPE_XGBE_ENA_HASH_UNI;
1134		reg |=  MPIPE_XGBE_COPY_ALL;
1135		gxio_mpipe_link_mac_wr(&priv->link, addr, reg);
1136	} else {
1137		addr = MPIPE_GBE_NETWORK_CONFIGURATION;
1138		reg  = gxio_mpipe_link_mac_rd(&priv->link, addr);
1139		reg &= ~MPIPE_GBE_ENA_MULTI_HASH;
1140		reg &= ~MPIPE_GBE_ENA_UNI_HASH;
1141		reg |=  MPIPE_GBE_COPY_ALL;
1142		gxio_mpipe_link_mac_wr(&priv->link, addr, reg);
1143	}
1144}
1145
1146static void
1147mpipe_promiscuous_disable(struct rte_eth_dev *dev)
1148{
1149	struct mpipe_dev_priv *priv = mpipe_priv(dev);
1150	int64_t reg;
1151	int addr;
1152
1153	if (priv->is_xaui) {
1154		addr = MPIPE_XAUI_RECEIVE_CONFIGURATION;
1155		reg  = gxio_mpipe_link_mac_rd(&priv->link, addr);
1156		reg |=  MPIPE_XGBE_ENA_HASH_MULTI;
1157		reg |=  MPIPE_XGBE_ENA_HASH_UNI;
1158		reg &= ~MPIPE_XGBE_COPY_ALL;
1159		gxio_mpipe_link_mac_wr(&priv->link, addr, reg);
1160	} else {
1161		addr = MPIPE_GBE_NETWORK_CONFIGURATION;
1162		reg  = gxio_mpipe_link_mac_rd(&priv->link, addr);
1163		reg |=  MPIPE_GBE_ENA_MULTI_HASH;
1164		reg |=  MPIPE_GBE_ENA_UNI_HASH;
1165		reg &= ~MPIPE_GBE_COPY_ALL;
1166		gxio_mpipe_link_mac_wr(&priv->link, addr, reg);
1167	}
1168}
1169
1170static const struct eth_dev_ops mpipe_dev_ops = {
1171	.dev_infos_get	         = mpipe_infos_get,
1172	.dev_configure	         = mpipe_configure,
1173	.dev_start	         = mpipe_start,
1174	.dev_stop	         = mpipe_stop,
1175	.dev_close	         = mpipe_close,
1176	.stats_get	         = mpipe_stats_get,
1177	.stats_reset	         = mpipe_stats_reset,
1178	.queue_stats_mapping_set = mpipe_queue_stats_mapping_set,
1179	.tx_queue_setup	         = mpipe_tx_queue_setup,
1180	.rx_queue_setup	         = mpipe_rx_queue_setup,
1181	.tx_queue_release	 = mpipe_tx_queue_release,
1182	.rx_queue_release	 = mpipe_rx_queue_release,
1183	.link_update	         = mpipe_link_update,
1184	.dev_set_link_up         = mpipe_set_link_up,
1185	.dev_set_link_down       = mpipe_set_link_down,
1186	.promiscuous_enable      = mpipe_promiscuous_enable,
1187	.promiscuous_disable     = mpipe_promiscuous_disable,
1188};
1189
1190static inline void
1191mpipe_xmit_null(struct mpipe_dev_priv *priv, int64_t start, int64_t end)
1192{
1193	gxio_mpipe_edesc_t null_desc = { { .bound = 1, .ns = 1 } };
1194	gxio_mpipe_equeue_t *equeue = &priv->equeue;
1195	int64_t slot;
1196
1197	for (slot = start; slot < end; slot++) {
1198		gxio_mpipe_equeue_put_at(equeue, null_desc, slot);
1199	}
1200}
1201
1202static void
1203mpipe_xmit_flush(struct mpipe_dev_priv *priv)
1204{
1205	gxio_mpipe_equeue_t *equeue = &priv->equeue;
1206	int64_t slot;
1207
1208	/* Post a dummy descriptor and wait for its return. */
1209	slot = gxio_mpipe_equeue_reserve(equeue, 1);
1210	if (slot < 0) {
1211		RTE_LOG(ERR, PMD, "%s: Failed to reserve stop slot.\n",
1212			mpipe_name(priv));
1213		return;
1214	}
1215
1216	mpipe_xmit_null(priv, slot, slot + 1);
1217
1218	while (!gxio_mpipe_equeue_is_complete(equeue, slot, 1)) {
1219		rte_pause();
1220	}
1221
1222	for (slot = 0; slot < priv->equeue_size; slot++) {
1223		if (priv->tx_comps[slot])
1224			rte_pktmbuf_free_seg(priv->tx_comps[slot]);
1225	}
1226}
1227
1228static void
1229mpipe_recv_flush(struct mpipe_dev_priv *priv)
1230{
1231	uint8_t in_port = priv->port_id;
1232	struct mpipe_rx_queue *rx_queue;
1233	gxio_mpipe_iqueue_t *iqueue;
1234	gxio_mpipe_idesc_t idesc;
1235	struct rte_mbuf *mbuf;
1236	unsigned queue;
1237
1238	/* Release packets on the buffer stack. */
1239	mpipe_recv_flush_stack(priv);
1240
1241	/* Flush packets sitting in recv queues. */
1242	for (queue = 0; queue < priv->nb_rx_queues; queue++) {
1243		rx_queue = mpipe_rx_queue(priv, queue);
1244		iqueue = &rx_queue->iqueue;
1245		while (gxio_mpipe_iqueue_try_get(iqueue, &idesc) >= 0) {
1246			/* Skip idesc with the 'buffer error' bit set. */
1247			if (idesc.be)
1248				continue;
1249			mbuf = mpipe_recv_mbuf(priv, &idesc, in_port);
1250			rte_pktmbuf_free(mbuf);
1251		}
1252		rte_free(rx_queue->rx_ring_mem);
1253	}
1254}
1255
1256static inline uint16_t
1257mpipe_do_xmit(struct mpipe_tx_queue *tx_queue, struct rte_mbuf **tx_pkts,
1258	      uint16_t nb_pkts)
1259{
1260	struct mpipe_dev_priv *priv = tx_queue->q.priv;
1261	gxio_mpipe_equeue_t *equeue = &priv->equeue;
1262	unsigned nb_bytes = 0;
1263	unsigned nb_sent = 0;
1264	int nb_slots, i;
1265	uint8_t port_id;
1266
1267	PMD_DEBUG_TX("Trying to transmit %d packets on %s:%d.\n",
1268		     nb_pkts, mpipe_name(tx_queue->q.priv),
1269		     tx_queue->q.queue_idx);
1270
1271	/* Optimistic assumption that we need exactly one slot per packet. */
1272	nb_slots = RTE_MIN(nb_pkts, MPIPE_TX_DESCS / 2);
1273
1274	do {
1275		struct rte_mbuf *mbuf = NULL, *pkt = NULL;
1276		int64_t slot;
1277
1278		/* Reserve eDMA ring slots. */
1279		slot = gxio_mpipe_equeue_try_reserve_fast(equeue, nb_slots);
1280		if (unlikely(slot < 0)) {
1281			break;
1282		}
1283
1284		for (i = 0; i < nb_slots; i++) {
1285			unsigned idx = (slot + i) & (priv->equeue_size - 1);
1286			rte_prefetch0(priv->tx_comps[idx]);
1287		}
1288
1289		/* Fill up slots with descriptor and completion info. */
1290		for (i = 0; i < nb_slots; i++) {
1291			unsigned idx = (slot + i) & (priv->equeue_size - 1);
1292			gxio_mpipe_edesc_t desc;
1293			struct rte_mbuf *next;
1294
1295			/* Starting on a new packet? */
1296			if (likely(!mbuf)) {
1297				int room = nb_slots - i;
1298
1299				pkt = mbuf = tx_pkts[nb_sent];
1300
1301				/* Bail out if we run out of descs. */
1302				if (unlikely(pkt->nb_segs > room))
1303					break;
1304
1305				nb_sent++;
1306			}
1307
1308			/* We have a segment to send. */
1309			next = mbuf->next;
1310
1311			if (priv->tx_comps[idx])
1312				rte_pktmbuf_free_seg(priv->tx_comps[idx]);
1313
1314			port_id = (mbuf->port < RTE_MAX_ETHPORTS) ?
1315						mbuf->port : priv->port_id;
1316			desc = (gxio_mpipe_edesc_t) { {
1317				.va        = rte_pktmbuf_mtod(mbuf, uintptr_t),
1318				.xfer_size = rte_pktmbuf_data_len(mbuf),
1319				.bound     = next ? 0 : 1,
1320				.stack_idx = mpipe_mbuf_stack_index(priv, mbuf),
1321				.size      = priv->rx_size_code,
1322			} };
1323			if (mpipe_local.mbuf_push_debt[port_id] > 0) {
1324				mpipe_local.mbuf_push_debt[port_id]--;
1325				desc.hwb = 1;
1326				priv->tx_comps[idx] = NULL;
1327			} else
1328				priv->tx_comps[idx] = mbuf;
1329
1330			nb_bytes += mbuf->data_len;
1331			gxio_mpipe_equeue_put_at(equeue, desc, slot + i);
1332
1333			PMD_DEBUG_TX("%s:%d: Sending packet %p, len %d\n",
1334				     mpipe_name(priv),
1335				     tx_queue->q.queue_idx,
1336				     rte_pktmbuf_mtod(mbuf, void *),
1337				     rte_pktmbuf_data_len(mbuf));
1338
1339			mbuf = next;
1340		}
1341
1342		if (unlikely(nb_sent < nb_pkts)) {
1343
1344			/* Fill remaining slots with null descriptors. */
1345			mpipe_xmit_null(priv, slot + i, slot + nb_slots);
1346
1347			/*
1348			 * Calculate exact number of descriptors needed for
1349			 * the next go around.
1350			 */
1351			nb_slots = 0;
1352			for (i = nb_sent; i < nb_pkts; i++) {
1353				nb_slots += tx_pkts[i]->nb_segs;
1354			}
1355
1356			nb_slots = RTE_MIN(nb_slots, MPIPE_TX_DESCS / 2);
1357		}
1358	} while (nb_sent < nb_pkts);
1359
1360	tx_queue->q.stats.packets += nb_sent;
1361	tx_queue->q.stats.bytes   += nb_bytes;
1362
1363	return nb_sent;
1364}
1365
1366static inline uint16_t
1367mpipe_do_recv(struct mpipe_rx_queue *rx_queue, struct rte_mbuf **rx_pkts,
1368	      uint16_t nb_pkts)
1369{
1370	struct mpipe_dev_priv *priv = rx_queue->q.priv;
1371	gxio_mpipe_iqueue_t *iqueue = &rx_queue->iqueue;
1372	gxio_mpipe_idesc_t *first_idesc, *idesc, *last_idesc;
1373	uint8_t in_port = rx_queue->q.port_id;
1374	const unsigned look_ahead = 8;
1375	int room = nb_pkts, rc = 0;
1376	unsigned nb_packets = 0;
1377	unsigned nb_dropped = 0;
1378	unsigned nb_nomem = 0;
1379	unsigned nb_bytes = 0;
1380	unsigned nb_descs, i;
1381
1382	while (room && !rc) {
1383		if (rx_queue->avail_descs < room) {
1384			rc = gxio_mpipe_iqueue_try_peek(iqueue,
1385							&rx_queue->next_desc);
1386			rx_queue->avail_descs = rc < 0 ? 0 : rc;
1387		}
1388
1389		if (unlikely(!rx_queue->avail_descs)) {
1390			break;
1391		}
1392
1393		nb_descs = RTE_MIN(room, rx_queue->avail_descs);
1394
1395		first_idesc = rx_queue->next_desc;
1396		last_idesc  = first_idesc + nb_descs;
1397
1398		rx_queue->next_desc   += nb_descs;
1399		rx_queue->avail_descs -= nb_descs;
1400
1401		for (i = 1; i < look_ahead; i++) {
1402			rte_prefetch0(first_idesc + i);
1403		}
1404
1405		PMD_DEBUG_RX("%s:%d: Trying to receive %d packets\n",
1406			     mpipe_name(rx_queue->q.priv),
1407			     rx_queue->q.queue_idx,
1408			     nb_descs);
1409
1410		for (idesc = first_idesc; idesc < last_idesc; idesc++) {
1411			struct rte_mbuf *mbuf;
1412
1413			PMD_DEBUG_RX("%s:%d: processing idesc %d/%d\n",
1414				     mpipe_name(priv),
1415				     rx_queue->q.queue_idx,
1416				     nb_packets, nb_descs);
1417
1418			rte_prefetch0(idesc + look_ahead);
1419
1420			PMD_DEBUG_RX("%s:%d: idesc %p, %s%s%s%s%s%s%s%s%s%s"
1421				     "size: %d, bkt: %d, chan: %d, ring: %d, sqn: %lu, va: %lu\n",
1422				     mpipe_name(priv),
1423				     rx_queue->q.queue_idx,
1424				     idesc,
1425				     idesc->me ? "me, " : "",
1426				     idesc->tr ? "tr, " : "",
1427				     idesc->ce ? "ce, " : "",
1428				     idesc->ct ? "ct, " : "",
1429				     idesc->cs ? "cs, " : "",
1430				     idesc->nr ? "nr, " : "",
1431				     idesc->sq ? "sq, " : "",
1432				     idesc->ts ? "ts, " : "",
1433				     idesc->ps ? "ps, " : "",
1434				     idesc->be ? "be, " : "",
1435				     idesc->l2_size,
1436				     idesc->bucket_id,
1437				     idesc->channel,
1438				     idesc->notif_ring,
1439				     (unsigned long)idesc->packet_sqn,
1440				     (unsigned long)idesc->va);
1441
1442			if (unlikely(gxio_mpipe_idesc_has_error(idesc))) {
1443				nb_dropped++;
1444				gxio_mpipe_iqueue_drop(iqueue, idesc);
1445				PMD_DEBUG_RX("%s:%d: Descriptor error\n",
1446					     mpipe_name(rx_queue->q.priv),
1447					     rx_queue->q.queue_idx);
1448				continue;
1449			}
1450
1451			if (mpipe_local.mbuf_push_debt[in_port] <
1452					MPIPE_BUF_DEBT_THRESHOLD)
1453				mpipe_local.mbuf_push_debt[in_port]++;
1454			else {
1455				mbuf = rte_mbuf_raw_alloc(priv->rx_mpool);
1456				if (unlikely(!mbuf)) {
1457					nb_nomem++;
1458					gxio_mpipe_iqueue_drop(iqueue, idesc);
1459					PMD_DEBUG_RX("%s:%d: alloc failure\n",
1460					     mpipe_name(rx_queue->q.priv),
1461					     rx_queue->q.queue_idx);
1462					continue;
1463				}
1464
1465				mpipe_recv_push(priv, mbuf);
1466			}
1467
1468			/* Get and setup the mbuf for the received packet. */
1469			mbuf = mpipe_recv_mbuf(priv, idesc, in_port);
1470
1471			/* Update results and statistics counters. */
1472			rx_pkts[nb_packets] = mbuf;
1473			nb_bytes += mbuf->pkt_len;
1474			nb_packets++;
1475		}
1476
1477		/*
1478		 * We release the ring in bursts, but do not track and release
1479		 * buckets.  This therefore breaks dynamic flow affinity, but
1480		 * we always operate in static affinity mode, and so we're OK
1481		 * with this optimization.
1482		 */
1483		gxio_mpipe_iqueue_advance(iqueue, nb_descs);
1484		gxio_mpipe_credit(iqueue->context, iqueue->ring, -1, nb_descs);
1485
1486		/*
1487		 * Go around once more if we haven't yet peeked the queue, and
1488		 * if we have more room to receive.
1489		 */
1490		room = nb_pkts - nb_packets;
1491	}
1492
1493	rx_queue->q.stats.packets += nb_packets;
1494	rx_queue->q.stats.bytes   += nb_bytes;
1495	rx_queue->q.stats.errors  += nb_dropped;
1496	rx_queue->q.stats.nomem   += nb_nomem;
1497
1498	PMD_DEBUG_RX("%s:%d: RX: %d/%d pkts/bytes, %d/%d drops/nomem\n",
1499		     mpipe_name(rx_queue->q.priv), rx_queue->q.queue_idx,
1500		     nb_packets, nb_bytes, nb_dropped, nb_nomem);
1501
1502	return nb_packets;
1503}
1504
1505static uint16_t
1506mpipe_recv_pkts(void *_rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
1507{
1508	struct mpipe_rx_queue *rx_queue = _rxq;
1509	uint16_t result = 0;
1510
1511	if (rx_queue) {
1512		mpipe_dp_enter(rx_queue->q.priv);
1513		if (likely(rx_queue->q.link_status))
1514			result = mpipe_do_recv(rx_queue, rx_pkts, nb_pkts);
1515		mpipe_dp_exit(rx_queue->q.priv);
1516	}
1517
1518	return result;
1519}
1520
1521static uint16_t
1522mpipe_xmit_pkts(void *_txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
1523{
1524	struct mpipe_tx_queue *tx_queue = _txq;
1525	uint16_t result = 0;
1526
1527	if (tx_queue) {
1528		mpipe_dp_enter(tx_queue->q.priv);
1529		if (likely(tx_queue->q.link_status))
1530			result = mpipe_do_xmit(tx_queue, tx_pkts, nb_pkts);
1531		mpipe_dp_exit(tx_queue->q.priv);
1532	}
1533
1534	return result;
1535}
1536
1537static int
1538mpipe_link_mac(const char *ifname, uint8_t *mac)
1539{
1540	int rc, idx;
1541	char name[GXIO_MPIPE_LINK_NAME_LEN];
1542
1543	for (idx = 0, rc = 0; !rc; idx++) {
1544		rc = gxio_mpipe_link_enumerate_mac(idx, name, mac);
1545		if (!rc && !strncmp(name, ifname, GXIO_MPIPE_LINK_NAME_LEN))
1546			return 0;
1547	}
1548	return -ENODEV;
1549}
1550
1551static int
1552rte_pmd_mpipe_devinit(const char *ifname,
1553		      const char *params __rte_unused)
1554{
1555	gxio_mpipe_context_t *context;
1556	struct rte_eth_dev *eth_dev;
1557	struct mpipe_dev_priv *priv;
1558	int instance, rc;
1559	uint8_t *mac;
1560
1561	/* Get the mPIPE instance that the device belongs to. */
1562	instance = gxio_mpipe_link_instance(ifname);
1563	context = mpipe_context(instance);
1564	if (!context) {
1565		RTE_LOG(ERR, PMD, "%s: No device for link.\n", ifname);
1566		return -ENODEV;
1567	}
1568
1569	priv = rte_zmalloc(NULL, sizeof(*priv), 0);
1570	if (!priv) {
1571		RTE_LOG(ERR, PMD, "%s: Failed to allocate priv.\n", ifname);
1572		return -ENOMEM;
1573	}
1574
1575	memset(&priv->tx_stat_mapping, 0xff, sizeof(priv->tx_stat_mapping));
1576	memset(&priv->rx_stat_mapping, 0xff, sizeof(priv->rx_stat_mapping));
1577	priv->context = context;
1578	priv->instance = instance;
1579	priv->is_xaui = (strncmp(ifname, "xgbe", 4) == 0);
1580	priv->channel = -1;
1581
1582	mac = priv->mac_addr.addr_bytes;
1583	rc = mpipe_link_mac(ifname, mac);
1584	if (rc < 0) {
1585		RTE_LOG(ERR, PMD, "%s: Failed to enumerate link.\n", ifname);
1586		rte_free(priv);
1587		return -ENODEV;
1588	}
1589
1590	eth_dev = rte_eth_dev_allocate(ifname, RTE_ETH_DEV_VIRTUAL);
1591	if (!eth_dev) {
1592		RTE_LOG(ERR, PMD, "%s: Failed to allocate device.\n", ifname);
1593		rte_free(priv);
1594		return -ENOMEM;
1595	}
1596
1597	RTE_LOG(INFO, PMD, "%s: Initialized mpipe device"
1598		"(mac %02x:%02x:%02x:%02x:%02x:%02x).\n",
1599		ifname, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
1600
1601	priv->eth_dev = eth_dev;
1602	priv->port_id = eth_dev->data->port_id;
1603	eth_dev->data->dev_private = priv;
1604	eth_dev->data->mac_addrs = &priv->mac_addr;
1605
1606	eth_dev->data->dev_flags = 0;
1607	eth_dev->data->kdrv = RTE_KDRV_NONE;
1608	eth_dev->driver = NULL;
1609	eth_dev->data->drv_name = drivername;
1610	eth_dev->data->numa_node = instance;
1611
1612	eth_dev->dev_ops      = &mpipe_dev_ops;
1613	eth_dev->rx_pkt_burst = &mpipe_recv_pkts;
1614	eth_dev->tx_pkt_burst = &mpipe_xmit_pkts;
1615
1616	rc = mpipe_link_init(priv);
1617	if (rc < 0) {
1618		RTE_LOG(ERR, PMD, "%s: Failed to init link.\n",
1619			mpipe_name(priv));
1620		return rc;
1621	}
1622
1623	return 0;
1624}
1625
1626static struct rte_driver pmd_mpipe_xgbe_drv = {
1627	.name = "xgbe",
1628	.type = PMD_VDEV,
1629	.init = rte_pmd_mpipe_devinit,
1630};
1631
1632static struct rte_driver pmd_mpipe_gbe_drv = {
1633	.name = "gbe",
1634	.type = PMD_VDEV,
1635	.init = rte_pmd_mpipe_devinit,
1636};
1637
1638PMD_REGISTER_DRIVER(pmd_mpipe_xgbe_drv);
1639PMD_REGISTER_DRIVER(pmd_mpipe_gbe_drv);
1640
1641static void __attribute__((constructor, used))
1642mpipe_init_contexts(void)
1643{
1644	struct mpipe_context *context;
1645	int rc, instance;
1646
1647	for (instance = 0; instance < GXIO_MPIPE_INSTANCE_MAX; instance++) {
1648		context = &mpipe_contexts[instance];
1649
1650		rte_spinlock_init(&context->lock);
1651		rc = gxio_mpipe_init(&context->context, instance);
1652		if (rc < 0)
1653			break;
1654	}
1655
1656	mpipe_instances = instance;
1657}
1658