mlx5.c revision 446954f6
1/*-
2 *   BSD LICENSE
3 *
4 *   Copyright 2015 6WIND S.A.
5 *   Copyright 2015 Mellanox.
6 *
7 *   Redistribution and use in source and binary forms, with or without
8 *   modification, are permitted provided that the following conditions
9 *   are met:
10 *
11 *     * Redistributions of source code must retain the above copyright
12 *       notice, this list of conditions and the following disclaimer.
13 *     * Redistributions in binary form must reproduce the above copyright
14 *       notice, this list of conditions and the following disclaimer in
15 *       the documentation and/or other materials provided with the
16 *       distribution.
17 *     * Neither the name of 6WIND S.A. nor the names of its
18 *       contributors may be used to endorse or promote products derived
19 *       from this software without specific prior written permission.
20 *
21 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34#include <stddef.h>
35#include <unistd.h>
36#include <string.h>
37#include <assert.h>
38#include <stdint.h>
39#include <stdlib.h>
40#include <errno.h>
41#include <net/if.h>
42
43/* Verbs header. */
44/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
45#ifdef PEDANTIC
46#pragma GCC diagnostic ignored "-pedantic"
47#endif
48#include <infiniband/verbs.h>
49#ifdef PEDANTIC
50#pragma GCC diagnostic error "-pedantic"
51#endif
52
53/* DPDK headers don't like -pedantic. */
54#ifdef PEDANTIC
55#pragma GCC diagnostic ignored "-pedantic"
56#endif
57#include <rte_malloc.h>
58#include <rte_ethdev.h>
59#include <rte_pci.h>
60#include <rte_common.h>
61#include <rte_kvargs.h>
62#ifdef PEDANTIC
63#pragma GCC diagnostic error "-pedantic"
64#endif
65
66#include "mlx5.h"
67#include "mlx5_utils.h"
68#include "mlx5_rxtx.h"
69#include "mlx5_autoconf.h"
70#include "mlx5_defs.h"
71
72/* Device parameter to enable RX completion queue compression. */
73#define MLX5_RXQ_CQE_COMP_EN "rxq_cqe_comp_en"
74
75/* Device parameter to configure inline send. */
76#define MLX5_TXQ_INLINE "txq_inline"
77
78/*
79 * Device parameter to configure the number of TX queues threshold for
80 * enabling inline send.
81 */
82#define MLX5_TXQS_MIN_INLINE "txqs_min_inline"
83
84/* Device parameter to enable multi-packet send WQEs. */
85#define MLX5_TXQ_MPW_EN "txq_mpw_en"
86
87/**
88 * Retrieve integer value from environment variable.
89 *
90 * @param[in] name
91 *   Environment variable name.
92 *
93 * @return
94 *   Integer value, 0 if the variable is not set.
95 */
96int
97mlx5_getenv_int(const char *name)
98{
99	const char *val = getenv(name);
100
101	if (val == NULL)
102		return 0;
103	return atoi(val);
104}
105
106/**
107 * DPDK callback to close the device.
108 *
109 * Destroy all queues and objects, free memory.
110 *
111 * @param dev
112 *   Pointer to Ethernet device structure.
113 */
114static void
115mlx5_dev_close(struct rte_eth_dev *dev)
116{
117	struct priv *priv = mlx5_get_priv(dev);
118	unsigned int i;
119
120	priv_lock(priv);
121	DEBUG("%p: closing device \"%s\"",
122	      (void *)dev,
123	      ((priv->ctx != NULL) ? priv->ctx->device->name : ""));
124	/* In case mlx5_dev_stop() has not been called. */
125	priv_dev_interrupt_handler_uninstall(priv, dev);
126	priv_special_flow_disable_all(priv);
127	priv_mac_addrs_disable(priv);
128	priv_destroy_hash_rxqs(priv);
129
130	/* Remove flow director elements. */
131	priv_fdir_disable(priv);
132	priv_fdir_delete_filters_list(priv);
133
134	/* Prevent crashes when queues are still in use. */
135	dev->rx_pkt_burst = removed_rx_burst;
136	dev->tx_pkt_burst = removed_tx_burst;
137	if (priv->rxqs != NULL) {
138		/* XXX race condition if mlx5_rx_burst() is still running. */
139		usleep(1000);
140		for (i = 0; (i != priv->rxqs_n); ++i) {
141			struct rxq *rxq = (*priv->rxqs)[i];
142			struct rxq_ctrl *rxq_ctrl;
143
144			if (rxq == NULL)
145				continue;
146			rxq_ctrl = container_of(rxq, struct rxq_ctrl, rxq);
147			(*priv->rxqs)[i] = NULL;
148			rxq_cleanup(rxq_ctrl);
149			rte_free(rxq_ctrl);
150		}
151		priv->rxqs_n = 0;
152		priv->rxqs = NULL;
153	}
154	if (priv->txqs != NULL) {
155		/* XXX race condition if mlx5_tx_burst() is still running. */
156		usleep(1000);
157		for (i = 0; (i != priv->txqs_n); ++i) {
158			struct txq *txq = (*priv->txqs)[i];
159			struct txq_ctrl *txq_ctrl;
160
161			if (txq == NULL)
162				continue;
163			txq_ctrl = container_of(txq, struct txq_ctrl, txq);
164			(*priv->txqs)[i] = NULL;
165			txq_cleanup(txq_ctrl);
166			rte_free(txq_ctrl);
167		}
168		priv->txqs_n = 0;
169		priv->txqs = NULL;
170	}
171	if (priv->pd != NULL) {
172		assert(priv->ctx != NULL);
173		claim_zero(ibv_dealloc_pd(priv->pd));
174		claim_zero(ibv_close_device(priv->ctx));
175	} else
176		assert(priv->ctx == NULL);
177	if (priv->rss_conf != NULL) {
178		for (i = 0; (i != hash_rxq_init_n); ++i)
179			rte_free((*priv->rss_conf)[i]);
180		rte_free(priv->rss_conf);
181	}
182	if (priv->reta_idx != NULL)
183		rte_free(priv->reta_idx);
184
185    mlx5_stats_free(dev);
186
187	priv_unlock(priv);
188	memset(priv, 0, sizeof(*priv));
189}
190
191static const struct eth_dev_ops mlx5_dev_ops = {
192	.dev_configure = mlx5_dev_configure,
193	.dev_start = mlx5_dev_start,
194	.dev_stop = mlx5_dev_stop,
195	.dev_set_link_down = mlx5_set_link_down,
196	.dev_set_link_up = mlx5_set_link_up,
197	.dev_close = mlx5_dev_close,
198	.promiscuous_enable = mlx5_promiscuous_enable,
199	.promiscuous_disable = mlx5_promiscuous_disable,
200	.allmulticast_enable = mlx5_allmulticast_enable,
201	.allmulticast_disable = mlx5_allmulticast_disable,
202	.link_update = mlx5_link_update,
203	.stats_get = mlx5_stats_get,
204	.stats_reset = mlx5_stats_reset,
205	.dev_infos_get = mlx5_dev_infos_get,
206	.dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get,
207	.vlan_filter_set = mlx5_vlan_filter_set,
208	.rx_queue_setup = mlx5_rx_queue_setup,
209	.tx_queue_setup = mlx5_tx_queue_setup,
210	.rx_queue_release = mlx5_rx_queue_release,
211	.tx_queue_release = mlx5_tx_queue_release,
212	.flow_ctrl_get = mlx5_dev_get_flow_ctrl,
213	.flow_ctrl_set = mlx5_dev_set_flow_ctrl,
214	.mac_addr_remove = mlx5_mac_addr_remove,
215	.mac_addr_add = mlx5_mac_addr_add,
216	.mac_addr_set = mlx5_mac_addr_set,
217	.mtu_set = mlx5_dev_set_mtu,
218	.vlan_strip_queue_set = mlx5_vlan_strip_queue_set,
219	.vlan_offload_set = mlx5_vlan_offload_set,
220	.reta_update = mlx5_dev_rss_reta_update,
221	.reta_query = mlx5_dev_rss_reta_query,
222	.rss_hash_update = mlx5_rss_hash_update,
223	.rss_hash_conf_get = mlx5_rss_hash_conf_get,
224	.filter_ctrl = mlx5_dev_filter_ctrl,
225};
226
227static struct {
228	struct rte_pci_addr pci_addr; /* associated PCI address */
229	uint32_t ports; /* physical ports bitfield. */
230} mlx5_dev[32];
231
232/**
233 * Get device index in mlx5_dev[] from PCI bus address.
234 *
235 * @param[in] pci_addr
236 *   PCI bus address to look for.
237 *
238 * @return
239 *   mlx5_dev[] index on success, -1 on failure.
240 */
241static int
242mlx5_dev_idx(struct rte_pci_addr *pci_addr)
243{
244	unsigned int i;
245	int ret = -1;
246
247	assert(pci_addr != NULL);
248	for (i = 0; (i != RTE_DIM(mlx5_dev)); ++i) {
249		if ((mlx5_dev[i].pci_addr.domain == pci_addr->domain) &&
250		    (mlx5_dev[i].pci_addr.bus == pci_addr->bus) &&
251		    (mlx5_dev[i].pci_addr.devid == pci_addr->devid) &&
252		    (mlx5_dev[i].pci_addr.function == pci_addr->function))
253			return i;
254		if ((mlx5_dev[i].ports == 0) && (ret == -1))
255			ret = i;
256	}
257	return ret;
258}
259
260/**
261 * Verify and store value for device argument.
262 *
263 * @param[in] key
264 *   Key argument to verify.
265 * @param[in] val
266 *   Value associated with key.
267 * @param opaque
268 *   User data.
269 *
270 * @return
271 *   0 on success, negative errno value on failure.
272 */
273static int
274mlx5_args_check(const char *key, const char *val, void *opaque)
275{
276	struct priv *priv = opaque;
277	unsigned long tmp;
278
279	errno = 0;
280	tmp = strtoul(val, NULL, 0);
281	if (errno) {
282		WARN("%s: \"%s\" is not a valid integer", key, val);
283		return errno;
284	}
285	if (strcmp(MLX5_RXQ_CQE_COMP_EN, key) == 0) {
286		priv->cqe_comp = !!tmp;
287	} else if (strcmp(MLX5_TXQ_INLINE, key) == 0) {
288		priv->txq_inline = tmp;
289	} else if (strcmp(MLX5_TXQS_MIN_INLINE, key) == 0) {
290		priv->txqs_inline = tmp;
291	} else if (strcmp(MLX5_TXQ_MPW_EN, key) == 0) {
292		priv->mps = !!tmp;
293	} else {
294		WARN("%s: unknown parameter", key);
295		return -EINVAL;
296	}
297	return 0;
298}
299
300/**
301 * Parse device parameters.
302 *
303 * @param priv
304 *   Pointer to private structure.
305 * @param devargs
306 *   Device arguments structure.
307 *
308 * @return
309 *   0 on success, errno value on failure.
310 */
311static int
312mlx5_args(struct priv *priv, struct rte_devargs *devargs)
313{
314	const char **params = (const char *[]){
315		MLX5_RXQ_CQE_COMP_EN,
316		MLX5_TXQ_INLINE,
317		MLX5_TXQS_MIN_INLINE,
318		MLX5_TXQ_MPW_EN,
319		NULL,
320	};
321	struct rte_kvargs *kvlist;
322	int ret = 0;
323	int i;
324
325	if (devargs == NULL)
326		return 0;
327	/* Following UGLY cast is done to pass checkpatch. */
328	kvlist = rte_kvargs_parse(devargs->args, params);
329	if (kvlist == NULL)
330		return 0;
331	/* Process parameters. */
332	for (i = 0; (params[i] != NULL); ++i) {
333		if (rte_kvargs_count(kvlist, params[i])) {
334			ret = rte_kvargs_process(kvlist, params[i],
335						 mlx5_args_check, priv);
336			if (ret != 0)
337				return ret;
338		}
339	}
340	rte_kvargs_free(kvlist);
341	return 0;
342}
343
344static struct eth_driver mlx5_driver;
345
346/**
347 * DPDK callback to register a PCI device.
348 *
349 * This function creates an Ethernet device for each port of a given
350 * PCI device.
351 *
352 * @param[in] pci_drv
353 *   PCI driver structure (mlx5_driver).
354 * @param[in] pci_dev
355 *   PCI device information.
356 *
357 * @return
358 *   0 on success, negative errno value on failure.
359 */
360static int
361mlx5_pci_devinit(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
362{
363	struct ibv_device **list;
364	struct ibv_device *ibv_dev;
365	int err = 0;
366	struct ibv_context *attr_ctx = NULL;
367	struct ibv_device_attr device_attr;
368	unsigned int sriov;
369	unsigned int mps;
370	int idx;
371	int i;
372
373	(void)pci_drv;
374	assert(pci_drv == &mlx5_driver.pci_drv);
375	/* Get mlx5_dev[] index. */
376	idx = mlx5_dev_idx(&pci_dev->addr);
377	if (idx == -1) {
378		ERROR("this driver cannot support any more adapters");
379		return -ENOMEM;
380	}
381	DEBUG("using driver device index %d", idx);
382
383	/* Save PCI address. */
384	mlx5_dev[idx].pci_addr = pci_dev->addr;
385	list = ibv_get_device_list(&i);
386	if (list == NULL) {
387		assert(errno);
388		if (errno == ENOSYS) {
389			WARN("cannot list devices, is ib_uverbs loaded?");
390			return 0;
391		}
392		return -errno;
393	}
394	assert(i >= 0);
395	/*
396	 * For each listed device, check related sysfs entry against
397	 * the provided PCI ID.
398	 */
399	while (i != 0) {
400		struct rte_pci_addr pci_addr;
401
402		--i;
403		DEBUG("checking device \"%s\"", list[i]->name);
404		if (mlx5_ibv_device_to_pci_addr(list[i], &pci_addr))
405			continue;
406		if ((pci_dev->addr.domain != pci_addr.domain) ||
407		    (pci_dev->addr.bus != pci_addr.bus) ||
408		    (pci_dev->addr.devid != pci_addr.devid) ||
409		    (pci_dev->addr.function != pci_addr.function))
410			continue;
411		sriov = ((pci_dev->id.device_id ==
412		       PCI_DEVICE_ID_MELLANOX_CONNECTX4VF) ||
413		      (pci_dev->id.device_id ==
414		       PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF));
415		/* Multi-packet send is only supported by ConnectX-4 Lx PF. */
416		mps = (pci_dev->id.device_id ==
417		       PCI_DEVICE_ID_MELLANOX_CONNECTX4LX);
418		INFO("PCI information matches, using device \"%s\""
419		     " (SR-IOV: %s, MPS: %s)",
420		     list[i]->name,
421		     sriov ? "true" : "false",
422		     mps ? "true" : "false");
423		attr_ctx = ibv_open_device(list[i]);
424		err = errno;
425		break;
426	}
427	if (attr_ctx == NULL) {
428		ibv_free_device_list(list);
429		switch (err) {
430		case 0:
431			WARN("cannot access device, is mlx5_ib loaded?");
432			return 0;
433		case EINVAL:
434			WARN("cannot use device, are drivers up to date?");
435			return 0;
436		}
437		assert(err > 0);
438		return -err;
439	}
440	ibv_dev = list[i];
441
442	DEBUG("device opened");
443	if (ibv_query_device(attr_ctx, &device_attr))
444		goto error;
445	INFO("%u port(s) detected", device_attr.phys_port_cnt);
446
447	for (i = 0; i < device_attr.phys_port_cnt; i++) {
448		uint32_t port = i + 1; /* ports are indexed from one */
449		uint32_t test = (1 << i);
450		struct ibv_context *ctx = NULL;
451		struct ibv_port_attr port_attr;
452		struct ibv_pd *pd = NULL;
453		struct priv *priv = NULL;
454		struct rte_eth_dev *eth_dev;
455		struct ibv_exp_device_attr exp_device_attr;
456		struct ether_addr mac;
457		uint16_t num_vfs = 0;
458
459		exp_device_attr.comp_mask =
460			IBV_EXP_DEVICE_ATTR_EXP_CAP_FLAGS |
461			IBV_EXP_DEVICE_ATTR_RX_HASH |
462			IBV_EXP_DEVICE_ATTR_VLAN_OFFLOADS |
463			IBV_EXP_DEVICE_ATTR_RX_PAD_END_ALIGN |
464			0;
465
466		DEBUG("using port %u (%08" PRIx32 ")", port, test);
467
468		ctx = ibv_open_device(ibv_dev);
469		if (ctx == NULL)
470			goto port_error;
471
472		/* Check port status. */
473		err = ibv_query_port(ctx, port, &port_attr);
474		if (err) {
475			ERROR("port query failed: %s", strerror(err));
476			goto port_error;
477		}
478
479		if (port_attr.link_layer != IBV_LINK_LAYER_ETHERNET) {
480			ERROR("port %d is not configured in Ethernet mode",
481			      port);
482			goto port_error;
483		}
484
485		if (port_attr.state != IBV_PORT_ACTIVE)
486			DEBUG("port %d is not active: \"%s\" (%d)",
487			      port, ibv_port_state_str(port_attr.state),
488			      port_attr.state);
489
490		/* Allocate protection domain. */
491		pd = ibv_alloc_pd(ctx);
492		if (pd == NULL) {
493			ERROR("PD allocation failure");
494			err = ENOMEM;
495			goto port_error;
496		}
497
498		mlx5_dev[idx].ports |= test;
499
500		/* from rte_ethdev.c */
501		priv = rte_zmalloc("ethdev private structure",
502				   sizeof(*priv),
503				   RTE_CACHE_LINE_SIZE);
504		if (priv == NULL) {
505			ERROR("priv allocation failure");
506			err = ENOMEM;
507			goto port_error;
508		}
509
510		priv->ctx = ctx;
511		priv->device_attr = device_attr;
512		priv->port = port;
513		priv->pd = pd;
514		priv->mtu = ETHER_MTU;
515		priv->mps = mps; /* Enable MPW by default if supported. */
516		priv->cqe_comp = 1; /* Enable compression by default. */
517
518
519        err = mlx5_args(priv, pci_dev->devargs);
520
521        /* TREX PATCH */
522        /* set for maximum performance default */
523        priv->txq_inline  =64;
524        priv->txqs_inline =4;
525
526
527		if (err) {
528			ERROR("failed to process device arguments: %s",
529			      strerror(err));
530			goto port_error;
531		}
532		if (ibv_exp_query_device(ctx, &exp_device_attr)) {
533			ERROR("ibv_exp_query_device() failed");
534			goto port_error;
535		}
536
537		priv->hw_csum =
538			((exp_device_attr.exp_device_cap_flags &
539			  IBV_EXP_DEVICE_RX_CSUM_TCP_UDP_PKT) &&
540			 (exp_device_attr.exp_device_cap_flags &
541			  IBV_EXP_DEVICE_RX_CSUM_IP_PKT));
542		DEBUG("checksum offloading is %ssupported",
543		      (priv->hw_csum ? "" : "not "));
544
545		priv->hw_csum_l2tun = !!(exp_device_attr.exp_device_cap_flags &
546					 IBV_EXP_DEVICE_VXLAN_SUPPORT);
547		DEBUG("L2 tunnel checksum offloads are %ssupported",
548		      (priv->hw_csum_l2tun ? "" : "not "));
549
550		priv->ind_table_max_size = exp_device_attr.rx_hash_caps.max_rwq_indirection_table_size;
551		/* Remove this check once DPDK supports larger/variable
552		 * indirection tables. */
553		if (priv->ind_table_max_size > (unsigned int)RSS_INDIRECTION_TABLE_SIZE)
554			priv->ind_table_max_size = RSS_INDIRECTION_TABLE_SIZE;
555		DEBUG("maximum RX indirection table size is %u",
556		      priv->ind_table_max_size);
557		priv->hw_vlan_strip = !!(exp_device_attr.wq_vlan_offloads_cap &
558					 IBV_EXP_RECEIVE_WQ_CVLAN_STRIP);
559		DEBUG("VLAN stripping is %ssupported",
560		      (priv->hw_vlan_strip ? "" : "not "));
561
562		priv->hw_fcs_strip = !!(exp_device_attr.exp_device_cap_flags &
563					IBV_EXP_DEVICE_SCATTER_FCS);
564		DEBUG("FCS stripping configuration is %ssupported",
565		      (priv->hw_fcs_strip ? "" : "not "));
566
567		priv->hw_padding = !!exp_device_attr.rx_pad_end_addr_align;
568		DEBUG("hardware RX end alignment padding is %ssupported",
569		      (priv->hw_padding ? "" : "not "));
570
571		priv_get_num_vfs(priv, &num_vfs);
572		priv->sriov = (num_vfs || sriov);
573		if (priv->mps && !mps) {
574			ERROR("multi-packet send not supported on this device"
575			      " (" MLX5_TXQ_MPW_EN ")");
576			err = ENOTSUP;
577			goto port_error;
578		}
579		/* Allocate and register default RSS hash keys. */
580		priv->rss_conf = rte_calloc(__func__, hash_rxq_init_n,
581					    sizeof((*priv->rss_conf)[0]), 0);
582		if (priv->rss_conf == NULL) {
583			err = ENOMEM;
584			goto port_error;
585		}
586		err = rss_hash_rss_conf_new_key(priv,
587						rss_hash_default_key,
588						rss_hash_default_key_len,
589						ETH_RSS_PROTO_MASK);
590		if (err)
591			goto port_error;
592		/* Configure the first MAC address by default. */
593		if (priv_get_mac(priv, &mac.addr_bytes)) {
594			ERROR("cannot get MAC address, is mlx5_en loaded?"
595			      " (errno: %s)", strerror(errno));
596			goto port_error;
597		}
598		INFO("port %u MAC address is %02x:%02x:%02x:%02x:%02x:%02x",
599		     priv->port,
600		     mac.addr_bytes[0], mac.addr_bytes[1],
601		     mac.addr_bytes[2], mac.addr_bytes[3],
602		     mac.addr_bytes[4], mac.addr_bytes[5]);
603		/* Register MAC address. */
604		claim_zero(priv_mac_addr_add(priv, 0,
605					     (const uint8_t (*)[ETHER_ADDR_LEN])
606					     mac.addr_bytes));
607		/* Initialize FD filters list. */
608		err = fdir_init_filters_list(priv);
609		if (err)
610			goto port_error;
611#ifndef NDEBUG
612		{
613			char ifname[IF_NAMESIZE];
614
615			if (priv_get_ifname(priv, &ifname) == 0)
616				DEBUG("port %u ifname is \"%s\"",
617				      priv->port, ifname);
618			else
619				DEBUG("port %u ifname is unknown", priv->port);
620		}
621#endif
622		/* Get actual MTU if possible. */
623		priv_get_mtu(priv, &priv->mtu);
624		DEBUG("port %u MTU is %u", priv->port, priv->mtu);
625
626		/* from rte_ethdev.c */
627		{
628			char name[RTE_ETH_NAME_MAX_LEN];
629
630			snprintf(name, sizeof(name), "%s port %u",
631				 ibv_get_device_name(ibv_dev), port);
632			eth_dev = rte_eth_dev_allocate(name, RTE_ETH_DEV_PCI);
633		}
634		if (eth_dev == NULL) {
635			ERROR("can not allocate rte ethdev");
636			err = ENOMEM;
637			goto port_error;
638		}
639
640		/* Secondary processes have to use local storage for their
641		 * private data as well as a copy of eth_dev->data, but this
642		 * pointer must not be modified before burst functions are
643		 * actually called. */
644		if (mlx5_is_secondary()) {
645			struct mlx5_secondary_data *sd =
646				&mlx5_secondary_data[eth_dev->data->port_id];
647			sd->primary_priv = eth_dev->data->dev_private;
648			if (sd->primary_priv == NULL) {
649				ERROR("no private data for port %u",
650						eth_dev->data->port_id);
651				err = EINVAL;
652				goto port_error;
653			}
654			sd->shared_dev_data = eth_dev->data;
655			rte_spinlock_init(&sd->lock);
656			memcpy(sd->data.name, sd->shared_dev_data->name,
657				   sizeof(sd->data.name));
658			sd->data.dev_private = priv;
659			sd->data.rx_mbuf_alloc_failed = 0;
660			sd->data.mtu = ETHER_MTU;
661			sd->data.port_id = sd->shared_dev_data->port_id;
662			sd->data.mac_addrs = priv->mac;
663			eth_dev->tx_pkt_burst = mlx5_tx_burst_secondary_setup;
664			eth_dev->rx_pkt_burst = mlx5_rx_burst_secondary_setup;
665		} else {
666			eth_dev->data->dev_private = priv;
667			eth_dev->data->rx_mbuf_alloc_failed = 0;
668			eth_dev->data->mtu = ETHER_MTU;
669			eth_dev->data->mac_addrs = priv->mac;
670		}
671
672		eth_dev->pci_dev = pci_dev;
673		rte_eth_copy_pci_info(eth_dev, pci_dev);
674		eth_dev->driver = &mlx5_driver;
675		priv->dev = eth_dev;
676		eth_dev->dev_ops = &mlx5_dev_ops;
677
678		TAILQ_INIT(&eth_dev->link_intr_cbs);
679
680		/* Bring Ethernet device up. */
681		DEBUG("forcing Ethernet interface up");
682		priv_set_flags(priv, ~IFF_UP, IFF_UP);
683		continue;
684
685port_error:
686		if (priv) {
687			rte_free(priv->rss_conf);
688			rte_free(priv);
689		}
690		if (pd)
691			claim_zero(ibv_dealloc_pd(pd));
692		if (ctx)
693			claim_zero(ibv_close_device(ctx));
694		break;
695	}
696
697	/*
698	 * XXX if something went wrong in the loop above, there is a resource
699	 * leak (ctx, pd, priv, dpdk ethdev) but we can do nothing about it as
700	 * long as the dpdk does not provide a way to deallocate a ethdev and a
701	 * way to enumerate the registered ethdevs to free the previous ones.
702	 */
703
704	/* no port found, complain */
705	if (!mlx5_dev[idx].ports) {
706		err = ENODEV;
707		goto error;
708	}
709
710error:
711	if (attr_ctx)
712		claim_zero(ibv_close_device(attr_ctx));
713	if (list)
714		ibv_free_device_list(list);
715	assert(err >= 0);
716	return -err;
717}
718
719static const struct rte_pci_id mlx5_pci_id_map[] = {
720	{
721		RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
722			       PCI_DEVICE_ID_MELLANOX_CONNECTX4)
723	},
724	{
725		RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
726			       PCI_DEVICE_ID_MELLANOX_CONNECTX4VF)
727	},
728	{
729		RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
730			       PCI_DEVICE_ID_MELLANOX_CONNECTX4LX)
731	},
732	{
733		RTE_PCI_DEVICE(PCI_VENDOR_ID_MELLANOX,
734			       PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF)
735	},
736	{
737		.vendor_id = 0
738	}
739};
740
741static struct eth_driver mlx5_driver = {
742	.pci_drv = {
743		.name = MLX5_DRIVER_NAME,
744		.id_table = mlx5_pci_id_map,
745		.devinit = mlx5_pci_devinit,
746		.drv_flags = RTE_PCI_DRV_INTR_LSC,
747	},
748	.dev_private_size = sizeof(struct priv)
749};
750
751/**
752 * Driver initialization routine.
753 */
754static int
755rte_mlx5_pmd_init(const char *name, const char *args)
756{
757	(void)name;
758	(void)args;
759	/*
760	 * RDMAV_HUGEPAGES_SAFE tells ibv_fork_init() we intend to use
761	 * huge pages. Calling ibv_fork_init() during init allows
762	 * applications to use fork() safely for purposes other than
763	 * using this PMD, which is not supported in forked processes.
764	 */
765	setenv("RDMAV_HUGEPAGES_SAFE", "1", 1);
766	ibv_fork_init();
767	rte_eal_pci_register(&mlx5_driver.pci_drv);
768	return 0;
769}
770
771static struct rte_driver rte_mlx5_driver = {
772	.type = PMD_PDEV,
773	.init = rte_mlx5_pmd_init,
774};
775
776PMD_REGISTER_DRIVER(rte_mlx5_driver, mlx5);
777DRIVER_REGISTER_PCI_TABLE(mlx5, mlx5_pci_id_map);
778