1/*-
2 *   BSD LICENSE
3 *
4 *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
5 *   All rights reserved.
6 *
7 *   Redistribution and use in source and binary forms, with or without
8 *   modification, are permitted provided that the following conditions
9 *   are met:
10 *
11 *     * Redistributions of source code must retain the above copyright
12 *       notice, this list of conditions and the following disclaimer.
13 *     * Redistributions in binary form must reproduce the above copyright
14 *       notice, this list of conditions and the following disclaimer in
15 *       the documentation and/or other materials provided with the
16 *       distribution.
17 *     * Neither the name of Intel Corporation nor the names of its
18 *       contributors may be used to endorse or promote products derived
19 *       from this software without specific prior written permission.
20 *
21 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34#include "rte_eth_ring.h"
35#include <rte_mbuf.h>
36#include <rte_ethdev.h>
37#include <rte_malloc.h>
38#include <rte_memcpy.h>
39#include <rte_memzone.h>
40#include <rte_string_fns.h>
41#include <rte_vdev.h>
42#include <rte_kvargs.h>
43#include <rte_errno.h>
44
45#define ETH_RING_NUMA_NODE_ACTION_ARG	"nodeaction"
46#define ETH_RING_ACTION_CREATE		"CREATE"
47#define ETH_RING_ACTION_ATTACH		"ATTACH"
48
49static const char *valid_arguments[] = {
50	ETH_RING_NUMA_NODE_ACTION_ARG,
51	NULL
52};
53
54enum dev_action {
55	DEV_CREATE,
56	DEV_ATTACH
57};
58
59struct ring_queue {
60	struct rte_ring *rng;
61	rte_atomic64_t rx_pkts;
62	rte_atomic64_t tx_pkts;
63	rte_atomic64_t err_pkts;
64};
65
66struct pmd_internals {
67	unsigned max_rx_queues;
68	unsigned max_tx_queues;
69
70	struct ring_queue rx_ring_queues[RTE_PMD_RING_MAX_RX_RINGS];
71	struct ring_queue tx_ring_queues[RTE_PMD_RING_MAX_TX_RINGS];
72
73	struct ether_addr address;
74	enum dev_action action;
75};
76
77
78static const char *drivername = "Rings PMD";
79static struct rte_eth_link pmd_link = {
80		.link_speed = ETH_SPEED_NUM_10G,
81		.link_duplex = ETH_LINK_FULL_DUPLEX,
82		.link_status = ETH_LINK_DOWN,
83		.link_autoneg = ETH_LINK_FIXED,
84};
85
86static uint16_t
87eth_ring_rx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs)
88{
89	void **ptrs = (void *)&bufs[0];
90	struct ring_queue *r = q;
91	const uint16_t nb_rx = (uint16_t)rte_ring_dequeue_burst(r->rng,
92			ptrs, nb_bufs);
93	if (r->rng->flags & RING_F_SC_DEQ)
94		r->rx_pkts.cnt += nb_rx;
95	else
96		rte_atomic64_add(&(r->rx_pkts), nb_rx);
97	return nb_rx;
98}
99
100static uint16_t
101eth_ring_tx(void *q, struct rte_mbuf **bufs, uint16_t nb_bufs)
102{
103	void **ptrs = (void *)&bufs[0];
104	struct ring_queue *r = q;
105	const uint16_t nb_tx = (uint16_t)rte_ring_enqueue_burst(r->rng,
106			ptrs, nb_bufs);
107	if (r->rng->flags & RING_F_SP_ENQ) {
108		r->tx_pkts.cnt += nb_tx;
109		r->err_pkts.cnt += nb_bufs - nb_tx;
110	} else {
111		rte_atomic64_add(&(r->tx_pkts), nb_tx);
112		rte_atomic64_add(&(r->err_pkts), nb_bufs - nb_tx);
113	}
114	return nb_tx;
115}
116
117static int
118eth_dev_configure(struct rte_eth_dev *dev __rte_unused) { return 0; }
119
120static int
121eth_dev_start(struct rte_eth_dev *dev)
122{
123	dev->data->dev_link.link_status = ETH_LINK_UP;
124	return 0;
125}
126
127static void
128eth_dev_stop(struct rte_eth_dev *dev)
129{
130	dev->data->dev_link.link_status = ETH_LINK_DOWN;
131}
132
133static int
134eth_dev_set_link_down(struct rte_eth_dev *dev)
135{
136	dev->data->dev_link.link_status = ETH_LINK_DOWN;
137	return 0;
138}
139
140static int
141eth_dev_set_link_up(struct rte_eth_dev *dev)
142{
143	dev->data->dev_link.link_status = ETH_LINK_UP;
144	return 0;
145}
146
147static int
148eth_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id,
149				    uint16_t nb_rx_desc __rte_unused,
150				    unsigned int socket_id __rte_unused,
151				    const struct rte_eth_rxconf *rx_conf __rte_unused,
152				    struct rte_mempool *mb_pool __rte_unused)
153{
154	struct pmd_internals *internals = dev->data->dev_private;
155	dev->data->rx_queues[rx_queue_id] = &internals->rx_ring_queues[rx_queue_id];
156	return 0;
157}
158
159static int
160eth_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id,
161				    uint16_t nb_tx_desc __rte_unused,
162				    unsigned int socket_id __rte_unused,
163				    const struct rte_eth_txconf *tx_conf __rte_unused)
164{
165	struct pmd_internals *internals = dev->data->dev_private;
166	dev->data->tx_queues[tx_queue_id] = &internals->tx_ring_queues[tx_queue_id];
167	return 0;
168}
169
170
171static void
172eth_dev_info(struct rte_eth_dev *dev,
173		struct rte_eth_dev_info *dev_info)
174{
175	struct pmd_internals *internals = dev->data->dev_private;
176	dev_info->driver_name = drivername;
177	dev_info->max_mac_addrs = 1;
178	dev_info->max_rx_pktlen = (uint32_t)-1;
179	dev_info->max_rx_queues = (uint16_t)internals->max_rx_queues;
180	dev_info->max_tx_queues = (uint16_t)internals->max_tx_queues;
181	dev_info->min_rx_bufsize = 0;
182	dev_info->pci_dev = NULL;
183}
184
185static void
186eth_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
187{
188	unsigned i;
189	unsigned long rx_total = 0, tx_total = 0, tx_err_total = 0;
190	const struct pmd_internals *internal = dev->data->dev_private;
191
192	for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS &&
193			i < dev->data->nb_rx_queues; i++) {
194		stats->q_ipackets[i] = internal->rx_ring_queues[i].rx_pkts.cnt;
195		rx_total += stats->q_ipackets[i];
196	}
197
198	for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS &&
199			i < dev->data->nb_tx_queues; i++) {
200		stats->q_opackets[i] = internal->tx_ring_queues[i].tx_pkts.cnt;
201		stats->q_errors[i] = internal->tx_ring_queues[i].err_pkts.cnt;
202		tx_total += stats->q_opackets[i];
203		tx_err_total += stats->q_errors[i];
204	}
205
206	stats->ipackets = rx_total;
207	stats->opackets = tx_total;
208	stats->oerrors = tx_err_total;
209}
210
211static void
212eth_stats_reset(struct rte_eth_dev *dev)
213{
214	unsigned i;
215	struct pmd_internals *internal = dev->data->dev_private;
216	for (i = 0; i < dev->data->nb_rx_queues; i++)
217		internal->rx_ring_queues[i].rx_pkts.cnt = 0;
218	for (i = 0; i < dev->data->nb_tx_queues; i++) {
219		internal->tx_ring_queues[i].tx_pkts.cnt = 0;
220		internal->tx_ring_queues[i].err_pkts.cnt = 0;
221	}
222}
223
224static void
225eth_mac_addr_remove(struct rte_eth_dev *dev __rte_unused,
226	uint32_t index __rte_unused)
227{
228}
229
230static void
231eth_mac_addr_add(struct rte_eth_dev *dev __rte_unused,
232	struct ether_addr *mac_addr __rte_unused,
233	uint32_t index __rte_unused,
234	uint32_t vmdq __rte_unused)
235{
236}
237
238static void
239eth_queue_release(void *q __rte_unused) { ; }
240static int
241eth_link_update(struct rte_eth_dev *dev __rte_unused,
242		int wait_to_complete __rte_unused) { return 0; }
243
244static const struct eth_dev_ops ops = {
245	.dev_start = eth_dev_start,
246	.dev_stop = eth_dev_stop,
247	.dev_set_link_up = eth_dev_set_link_up,
248	.dev_set_link_down = eth_dev_set_link_down,
249	.dev_configure = eth_dev_configure,
250	.dev_infos_get = eth_dev_info,
251	.rx_queue_setup = eth_rx_queue_setup,
252	.tx_queue_setup = eth_tx_queue_setup,
253	.rx_queue_release = eth_queue_release,
254	.tx_queue_release = eth_queue_release,
255	.link_update = eth_link_update,
256	.stats_get = eth_stats_get,
257	.stats_reset = eth_stats_reset,
258	.mac_addr_remove = eth_mac_addr_remove,
259	.mac_addr_add = eth_mac_addr_add,
260};
261
262static int
263do_eth_dev_ring_create(const char *name,
264		struct rte_ring * const rx_queues[], const unsigned nb_rx_queues,
265		struct rte_ring *const tx_queues[], const unsigned nb_tx_queues,
266		const unsigned numa_node, enum dev_action action)
267{
268	struct rte_eth_dev_data *data = NULL;
269	struct pmd_internals *internals = NULL;
270	struct rte_eth_dev *eth_dev = NULL;
271	unsigned i;
272
273	RTE_LOG(INFO, PMD, "Creating rings-backed ethdev on numa socket %u\n",
274			numa_node);
275
276	/* now do all data allocation - for eth_dev structure, dummy pci driver
277	 * and internal (private) data
278	 */
279	data = rte_zmalloc_socket(name, sizeof(*data), 0, numa_node);
280	if (data == NULL) {
281		rte_errno = ENOMEM;
282		goto error;
283	}
284
285	data->rx_queues = rte_zmalloc_socket(name,
286			sizeof(void *) * nb_rx_queues, 0, numa_node);
287	if (data->rx_queues == NULL) {
288		rte_errno = ENOMEM;
289		goto error;
290	}
291
292	data->tx_queues = rte_zmalloc_socket(name,
293			sizeof(void *) * nb_tx_queues, 0, numa_node);
294	if (data->tx_queues == NULL) {
295		rte_errno = ENOMEM;
296		goto error;
297	}
298
299	internals = rte_zmalloc_socket(name, sizeof(*internals), 0, numa_node);
300	if (internals == NULL) {
301		rte_errno = ENOMEM;
302		goto error;
303	}
304
305	/* reserve an ethdev entry */
306	eth_dev = rte_eth_dev_allocate(name);
307	if (eth_dev == NULL) {
308		rte_errno = ENOSPC;
309		goto error;
310	}
311
312	/* now put it all together
313	 * - store queue data in internals,
314	 * - store numa_node info in eth_dev_data
315	 * - point eth_dev_data to internals
316	 * - and point eth_dev structure to new eth_dev_data structure
317	 */
318	/* NOTE: we'll replace the data element, of originally allocated eth_dev
319	 * so the rings are local per-process */
320
321	internals->action = action;
322	internals->max_rx_queues = nb_rx_queues;
323	internals->max_tx_queues = nb_tx_queues;
324	for (i = 0; i < nb_rx_queues; i++) {
325		internals->rx_ring_queues[i].rng = rx_queues[i];
326		data->rx_queues[i] = &internals->rx_ring_queues[i];
327	}
328	for (i = 0; i < nb_tx_queues; i++) {
329		internals->tx_ring_queues[i].rng = tx_queues[i];
330		data->tx_queues[i] = &internals->tx_ring_queues[i];
331	}
332
333	data->dev_private = internals;
334	data->port_id = eth_dev->data->port_id;
335	memmove(data->name, eth_dev->data->name, sizeof(data->name));
336	data->nb_rx_queues = (uint16_t)nb_rx_queues;
337	data->nb_tx_queues = (uint16_t)nb_tx_queues;
338	data->dev_link = pmd_link;
339	data->mac_addrs = &internals->address;
340
341	eth_dev->data = data;
342	eth_dev->driver = NULL;
343	eth_dev->dev_ops = &ops;
344	data->dev_flags = RTE_ETH_DEV_DETACHABLE;
345	data->kdrv = RTE_KDRV_NONE;
346	data->drv_name = drivername;
347	data->numa_node = numa_node;
348
349	TAILQ_INIT(&(eth_dev->link_intr_cbs));
350
351	/* finally assign rx and tx ops */
352	eth_dev->rx_pkt_burst = eth_ring_rx;
353	eth_dev->tx_pkt_burst = eth_ring_tx;
354
355	return data->port_id;
356
357error:
358	if (data) {
359		rte_free(data->rx_queues);
360		rte_free(data->tx_queues);
361	}
362	rte_free(data);
363	rte_free(internals);
364
365	return -1;
366}
367
368int
369rte_eth_from_rings(const char *name, struct rte_ring *const rx_queues[],
370		const unsigned nb_rx_queues,
371		struct rte_ring *const tx_queues[],
372		const unsigned nb_tx_queues,
373		const unsigned numa_node)
374{
375	/* do some parameter checking */
376	if (rx_queues == NULL && nb_rx_queues > 0) {
377		rte_errno = EINVAL;
378		return -1;
379	}
380	if (tx_queues == NULL && nb_tx_queues > 0) {
381		rte_errno = EINVAL;
382		return -1;
383	}
384	if (nb_rx_queues > RTE_PMD_RING_MAX_RX_RINGS) {
385		rte_errno = EINVAL;
386		return -1;
387	}
388
389	return do_eth_dev_ring_create(name, rx_queues, nb_rx_queues,
390			tx_queues, nb_tx_queues, numa_node, DEV_ATTACH);
391}
392
393int
394rte_eth_from_ring(struct rte_ring *r)
395{
396	return rte_eth_from_rings(r->name, &r, 1, &r, 1,
397			r->memzone ? r->memzone->socket_id : SOCKET_ID_ANY);
398}
399
400static int
401eth_dev_ring_create(const char *name, const unsigned numa_node,
402		enum dev_action action)
403{
404	/* rx and tx are so-called from point of view of first port.
405	 * They are inverted from the point of view of second port
406	 */
407	struct rte_ring *rxtx[RTE_PMD_RING_MAX_RX_RINGS];
408	unsigned i;
409	char rng_name[RTE_RING_NAMESIZE];
410	unsigned num_rings = RTE_MIN(RTE_PMD_RING_MAX_RX_RINGS,
411			RTE_PMD_RING_MAX_TX_RINGS);
412
413	for (i = 0; i < num_rings; i++) {
414		snprintf(rng_name, sizeof(rng_name), "ETH_RXTX%u_%s", i, name);
415		rxtx[i] = (action == DEV_CREATE) ?
416				rte_ring_create(rng_name, 1024, numa_node,
417						RING_F_SP_ENQ|RING_F_SC_DEQ) :
418				rte_ring_lookup(rng_name);
419		if (rxtx[i] == NULL)
420			return -1;
421	}
422
423	if (do_eth_dev_ring_create(name, rxtx, num_rings, rxtx, num_rings,
424		numa_node, action) < 0)
425		return -1;
426
427	return 0;
428}
429
430struct node_action_pair {
431	char name[PATH_MAX];
432	unsigned node;
433	enum dev_action action;
434};
435
436struct node_action_list {
437	unsigned total;
438	unsigned count;
439	struct node_action_pair *list;
440};
441
442static int parse_kvlist (const char *key __rte_unused, const char *value, void *data)
443{
444	struct node_action_list *info = data;
445	int ret;
446	char *name;
447	char *action;
448	char *node;
449	char *end;
450
451	name = strdup(value);
452
453	ret = -EINVAL;
454
455	if (!name) {
456		RTE_LOG(WARNING, PMD, "command line paramter is empty for ring pmd!\n");
457		goto out;
458	}
459
460	node = strchr(name, ':');
461	if (!node) {
462		RTE_LOG(WARNING, PMD, "could not parse node value from %s", name);
463		goto out;
464	}
465
466	*node = '\0';
467	node++;
468
469	action = strchr(node, ':');
470	if (!action) {
471		RTE_LOG(WARNING, PMD, "could not action value from %s", node);
472		goto out;
473	}
474
475	*action = '\0';
476	action++;
477
478	/*
479	 * Need to do some sanity checking here
480	 */
481
482	if (strcmp(action, ETH_RING_ACTION_ATTACH) == 0)
483		info->list[info->count].action = DEV_ATTACH;
484	else if (strcmp(action, ETH_RING_ACTION_CREATE) == 0)
485		info->list[info->count].action = DEV_CREATE;
486	else
487		goto out;
488
489	errno = 0;
490	info->list[info->count].node = strtol(node, &end, 10);
491
492	if ((errno != 0) || (*end != '\0')) {
493		RTE_LOG(WARNING, PMD, "node value %s is unparseable as a number\n", node);
494		goto out;
495	}
496
497	snprintf(info->list[info->count].name, sizeof(info->list[info->count].name), "%s", name);
498
499	info->count++;
500
501	ret = 0;
502out:
503	free(name);
504	return ret;
505}
506
507static int
508rte_pmd_ring_probe(const char *name, const char *params)
509{
510	struct rte_kvargs *kvlist = NULL;
511	int ret = 0;
512	struct node_action_list *info = NULL;
513
514	RTE_LOG(INFO, PMD, "Initializing pmd_ring for %s\n", name);
515
516	if (params == NULL || params[0] == '\0') {
517		ret = eth_dev_ring_create(name, rte_socket_id(), DEV_CREATE);
518		if (ret == -1) {
519			RTE_LOG(INFO, PMD,
520				"Attach to pmd_ring for %s\n", name);
521			ret = eth_dev_ring_create(name, rte_socket_id(),
522						  DEV_ATTACH);
523		}
524	}
525	else {
526		kvlist = rte_kvargs_parse(params, valid_arguments);
527
528		if (!kvlist) {
529			RTE_LOG(INFO, PMD, "Ignoring unsupported parameters when creating"
530					" rings-backed ethernet device\n");
531			ret = eth_dev_ring_create(name, rte_socket_id(),
532						  DEV_CREATE);
533			if (ret == -1) {
534				RTE_LOG(INFO, PMD,
535					"Attach to pmd_ring for %s\n",
536					name);
537				ret = eth_dev_ring_create(name, rte_socket_id(),
538							  DEV_ATTACH);
539			}
540			return ret;
541		} else {
542			ret = rte_kvargs_count(kvlist, ETH_RING_NUMA_NODE_ACTION_ARG);
543			info = rte_zmalloc("struct node_action_list",
544					   sizeof(struct node_action_list) +
545					   (sizeof(struct node_action_pair) * ret),
546					   0);
547			if (!info)
548				goto out_free;
549
550			info->total = ret;
551			info->list = (struct node_action_pair*)(info + 1);
552
553			ret = rte_kvargs_process(kvlist, ETH_RING_NUMA_NODE_ACTION_ARG,
554						 parse_kvlist, info);
555
556			if (ret < 0)
557				goto out_free;
558
559			for (info->count = 0; info->count < info->total; info->count++) {
560				ret = eth_dev_ring_create(info->list[info->count].name,
561							  info->list[info->count].node,
562							  info->list[info->count].action);
563				if ((ret == -1) &&
564				    (info->list[info->count].action == DEV_CREATE)) {
565					RTE_LOG(INFO, PMD,
566						"Attach to pmd_ring for %s\n",
567						name);
568					ret = eth_dev_ring_create(name,
569							info->list[info->count].node,
570							DEV_ATTACH);
571				}
572			}
573		}
574	}
575
576out_free:
577	rte_kvargs_free(kvlist);
578	rte_free(info);
579	return ret;
580}
581
582static int
583rte_pmd_ring_remove(const char *name)
584{
585	struct rte_eth_dev *eth_dev = NULL;
586	struct pmd_internals *internals = NULL;
587	struct ring_queue *r = NULL;
588	uint16_t i;
589
590	RTE_LOG(INFO, PMD, "Un-Initializing pmd_ring for %s\n", name);
591
592	if (name == NULL)
593		return -EINVAL;
594
595	/* find an ethdev entry */
596	eth_dev = rte_eth_dev_allocated(name);
597	if (eth_dev == NULL)
598		return -ENODEV;
599
600	eth_dev_stop(eth_dev);
601
602	internals = eth_dev->data->dev_private;
603	if (internals->action == DEV_CREATE) {
604		/*
605		 * it is only necessary to delete the rings in rx_queues because
606		 * they are the same used in tx_queues
607		 */
608		for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
609			r = eth_dev->data->rx_queues[i];
610			rte_ring_free(r->rng);
611		}
612	}
613
614	rte_free(eth_dev->data->rx_queues);
615	rte_free(eth_dev->data->tx_queues);
616	rte_free(eth_dev->data->dev_private);
617
618	rte_free(eth_dev->data);
619
620	rte_eth_dev_release_port(eth_dev);
621	return 0;
622}
623
624static struct rte_vdev_driver pmd_ring_drv = {
625	.probe = rte_pmd_ring_probe,
626	.remove = rte_pmd_ring_remove,
627};
628
629RTE_PMD_REGISTER_VDEV(net_ring, pmd_ring_drv);
630RTE_PMD_REGISTER_ALIAS(net_ring, eth_ring);
631RTE_PMD_REGISTER_PARAM_STRING(net_ring,
632	ETH_RING_NUMA_NODE_ACTION_ARG "=name:node:action(ATTACH|CREATE)");
633