mlx5_fdir.c revision 54305984
1/*-
2 *   BSD LICENSE
3 *
4 *   Copyright 2015 6WIND S.A.
5 *   Copyright 2015 Mellanox.
6 *
7 *   Redistribution and use in source and binary forms, with or without
8 *   modification, are permitted provided that the following conditions
9 *   are met:
10 *
11 *     * Redistributions of source code must retain the above copyright
12 *       notice, this list of conditions and the following disclaimer.
13 *     * Redistributions in binary form must reproduce the above copyright
14 *       notice, this list of conditions and the following disclaimer in
15 *       the documentation and/or other materials provided with the
16 *       distribution.
17 *     * Neither the name of 6WIND S.A. nor the names of its
18 *       contributors may be used to endorse or promote products derived
19 *       from this software without specific prior written permission.
20 *
21 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34#include <stddef.h>
35#include <assert.h>
36#include <stdint.h>
37#include <string.h>
38#include <errno.h>
39
40/* Verbs header. */
41/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
42#ifdef PEDANTIC
43#pragma GCC diagnostic ignored "-pedantic"
44#endif
45#include <infiniband/verbs_exp.h>
46#ifdef PEDANTIC
47#pragma GCC diagnostic error "-pedantic"
48#endif
49
50/* DPDK headers don't like -pedantic. */
51#ifdef PEDANTIC
52#pragma GCC diagnostic ignored "-pedantic"
53#endif
54#include <rte_ether.h>
55#include <rte_malloc.h>
56#include <rte_ethdev.h>
57#include <rte_common.h>
58#ifdef PEDANTIC
59#pragma GCC diagnostic error "-pedantic"
60#endif
61
62#include "mlx5.h"
63#include "mlx5_rxtx.h"
64
65struct fdir_flow_desc {
66	uint16_t dst_port;
67	uint16_t src_port;
68	uint32_t src_ip[4];
69	uint32_t dst_ip[4];
70    uint8_t  tos;
71    uint8_t  ip_id;
72    uint16_t proto;
73
74	uint8_t	mac[6];
75	uint16_t vlan_tag;
76	enum hash_rxq_type type;
77};
78
79struct mlx5_fdir_filter {
80	LIST_ENTRY(mlx5_fdir_filter) next;
81	uint16_t queue; /* Queue assigned to if FDIR match. */
82	struct fdir_flow_desc desc;
83	struct ibv_exp_flow *flow;
84};
85
86LIST_HEAD(fdir_filter_list, mlx5_fdir_filter);
87
88/**
89 * Convert struct rte_eth_fdir_filter to mlx5 filter descriptor.
90 *
91 * @param[in] fdir_filter
92 *   DPDK filter structure to convert.
93 * @param[out] desc
94 *   Resulting mlx5 filter descriptor.
95 * @param mode
96 *   Flow director mode.
97 */
98static void
99fdir_filter_to_flow_desc(const struct rte_eth_fdir_filter *fdir_filter,
100			 struct fdir_flow_desc *desc, enum rte_fdir_mode mode)
101{
102	/* Initialize descriptor. */
103	memset(desc, 0, sizeof(*desc));
104
105	/* Set VLAN ID. */
106	desc->vlan_tag = fdir_filter->input.flow_ext.vlan_tci;
107
108	/* Set MAC address. */
109	if (mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
110		rte_memcpy(desc->mac,
111			   fdir_filter->input.flow.mac_vlan_flow.mac_addr.
112				addr_bytes,
113			   sizeof(desc->mac));
114		desc->type = HASH_RXQ_ETH;
115		return;
116	}
117
118	/* Set mode */
119	switch (fdir_filter->input.flow_type) {
120	case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
121		desc->type = HASH_RXQ_UDPV4;
122		break;
123	case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
124		desc->type = HASH_RXQ_TCPV4;
125		break;
126	case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
127		desc->type = HASH_RXQ_IPV4;
128		break;
129	case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
130		desc->type = HASH_RXQ_UDPV6;
131		break;
132	case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
133		desc->type = HASH_RXQ_TCPV6;
134		break;
135	case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
136		desc->type = HASH_RXQ_IPV6;
137		break;
138	default:
139		break;
140	}
141
142	/* Set flow values */
143	switch (fdir_filter->input.flow_type) {
144	case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
145	case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
146		desc->src_port = fdir_filter->input.flow.udp4_flow.src_port;
147		desc->dst_port = fdir_filter->input.flow.udp4_flow.dst_port;
148
149	case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
150		desc->src_ip[0] = fdir_filter->input.flow.ip4_flow.src_ip;
151		desc->dst_ip[0] = fdir_filter->input.flow.ip4_flow.dst_ip;
152        desc->tos       = fdir_filter->input.flow.ip4_flow.ttl; /* TTL is map to TOS*/
153        desc->ip_id     = fdir_filter->input.flow.ip4_flow.ip_id;
154        desc->proto     = fdir_filter->input.flow.ip4_flow.proto;
155		break;
156	case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
157	case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
158		desc->src_port = fdir_filter->input.flow.udp6_flow.src_port;
159		desc->dst_port = fdir_filter->input.flow.udp6_flow.dst_port;
160		/* Fall through. */
161	case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
162		rte_memcpy(desc->src_ip,
163			   fdir_filter->input.flow.ipv6_flow.src_ip,
164			   sizeof(desc->src_ip));
165		rte_memcpy(desc->dst_ip,
166			   fdir_filter->input.flow.ipv6_flow.dst_ip,
167			   sizeof(desc->dst_ip));
168        desc->tos       = fdir_filter->input.flow.ipv6_flow.tc;
169        desc->ip_id     = (uint8_t)fdir_filter->input.flow.ipv6_flow.flow_label;
170        desc->proto     = fdir_filter->input.flow.ipv6_flow.proto;
171
172		break;
173	default:
174		break;
175	}
176}
177
178/**
179 * Check if two flow descriptors overlap according to configured mask.
180 *
181 * @param priv
182 *   Private structure that provides flow director mask.
183 * @param desc1
184 *   First flow descriptor to compare.
185 * @param desc2
186 *   Second flow descriptor to compare.
187 *
188 * @return
189 *   Nonzero if descriptors overlap.
190 */
191static int
192priv_fdir_overlap(const struct priv *priv,
193		  const struct fdir_flow_desc *desc1,
194		  const struct fdir_flow_desc *desc2)
195{
196	const struct rte_eth_fdir_masks *mask =
197		&priv->dev->data->dev_conf.fdir_conf.mask;
198	unsigned int i;
199
200	if (desc1->type != desc2->type)
201		return 0;
202	/* Ignore non masked bits. */
203	for (i = 0; i != RTE_DIM(desc1->mac); ++i)
204		if ((desc1->mac[i] & mask->mac_addr_byte_mask) !=
205		    (desc2->mac[i] & mask->mac_addr_byte_mask))
206			return 0;
207	if (((desc1->src_port & mask->src_port_mask) !=
208	     (desc2->src_port & mask->src_port_mask)) ||
209	    ((desc1->dst_port & mask->dst_port_mask) !=
210	     (desc2->dst_port & mask->dst_port_mask)))
211		return 0;
212	switch (desc1->type) {
213	case HASH_RXQ_IPV4:
214	case HASH_RXQ_UDPV4:
215	case HASH_RXQ_TCPV4:
216		if (((desc1->src_ip[0] & mask->ipv4_mask.src_ip) !=
217		     (desc2->src_ip[0] & mask->ipv4_mask.src_ip)) ||
218		    ((desc1->dst_ip[0] & mask->ipv4_mask.dst_ip) !=
219		     (desc2->dst_ip[0] & mask->ipv4_mask.dst_ip)))
220			return 0;
221		break;
222	case HASH_RXQ_IPV6:
223	case HASH_RXQ_UDPV6:
224	case HASH_RXQ_TCPV6:
225		for (i = 0; i != RTE_DIM(desc1->src_ip); ++i)
226			if (((desc1->src_ip[i] & mask->ipv6_mask.src_ip[i]) !=
227			     (desc2->src_ip[i] & mask->ipv6_mask.src_ip[i])) ||
228			    ((desc1->dst_ip[i] & mask->ipv6_mask.dst_ip[i]) !=
229			     (desc2->dst_ip[i] & mask->ipv6_mask.dst_ip[i])))
230				return 0;
231		break;
232	default:
233		break;
234	}
235	return 1;
236}
237
238/**
239 * Create flow director steering rule for a specific filter.
240 *
241 * @param priv
242 *   Private structure.
243 * @param mlx5_fdir_filter
244 *   Filter to create a steering rule for.
245 * @param fdir_queue
246 *   Flow director queue for matching packets.
247 *
248 * @return
249 *   0 on success, errno value on failure.
250 */
251static int
252priv_fdir_flow_add(struct priv *priv,
253		   struct mlx5_fdir_filter *mlx5_fdir_filter,
254		   struct fdir_queue *fdir_queue)
255{
256	struct ibv_exp_flow *flow;
257	struct fdir_flow_desc *desc = &mlx5_fdir_filter->desc;
258	enum rte_fdir_mode fdir_mode =
259		priv->dev->data->dev_conf.fdir_conf.mode;
260	struct rte_eth_fdir_masks *mask =
261		&priv->dev->data->dev_conf.fdir_conf.mask;
262	FLOW_ATTR_SPEC_ETH(data, priv_flow_attr(priv, NULL, 0, desc->type));
263	struct ibv_exp_flow_attr *attr = &data->attr;
264	uintptr_t spec_offset = (uintptr_t)&data->spec;
265	struct ibv_exp_flow_spec_eth *spec_eth;
266	struct ibv_exp_flow_spec_ipv4_ext *spec_ipv4;
267	struct ibv_exp_flow_spec_ipv6_ext *spec_ipv6;
268	struct ibv_exp_flow_spec_tcp_udp *spec_tcp_udp;
269	struct mlx5_fdir_filter *iter_fdir_filter;
270	unsigned int i;
271
272	/* Abort if an existing flow overlaps this one to avoid packet
273	 * duplication, even if it targets another queue. */
274	LIST_FOREACH(iter_fdir_filter, priv->fdir_filter_list, next)
275		if ((iter_fdir_filter != mlx5_fdir_filter) &&
276		    (iter_fdir_filter->flow != NULL) &&
277		    (priv_fdir_overlap(priv,
278				       &mlx5_fdir_filter->desc,
279				       &iter_fdir_filter->desc)))
280			return EEXIST;
281
282	/*
283	 * No padding must be inserted by the compiler between attr and spec.
284	 * This layout is expected by libibverbs.
285	 */
286	assert(((uint8_t *)attr + sizeof(*attr)) == (uint8_t *)spec_offset);
287	priv_flow_attr(priv, attr, sizeof(data), desc->type);
288
289	/* Set Ethernet spec */
290	spec_eth = (struct ibv_exp_flow_spec_eth *)spec_offset;
291
292	/* The first specification must be Ethernet. */
293	assert(spec_eth->type == IBV_EXP_FLOW_SPEC_ETH);
294	assert(spec_eth->size == sizeof(*spec_eth));
295
296	/* VLAN ID */
297	spec_eth->val.vlan_tag = desc->vlan_tag & mask->vlan_tci_mask;
298	spec_eth->mask.vlan_tag = mask->vlan_tci_mask;
299
300	/* Update priority */
301	attr->priority = 2;
302
303	if (fdir_mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
304		/* MAC Address */
305		for (i = 0; i != RTE_DIM(spec_eth->mask.dst_mac); ++i) {
306			spec_eth->val.dst_mac[i] =
307				desc->mac[i] & mask->mac_addr_byte_mask;
308			spec_eth->mask.dst_mac[i] = mask->mac_addr_byte_mask;
309		}
310		goto create_flow;
311	}
312
313	switch (desc->type) {
314	case HASH_RXQ_IPV4:
315	case HASH_RXQ_UDPV4:
316	case HASH_RXQ_TCPV4:
317		spec_offset += spec_eth->size;
318
319		/* Set IP spec */
320		spec_ipv4 = (struct ibv_exp_flow_spec_ipv4_ext *)spec_offset;
321
322		/* The second specification must be IP. */
323		assert(spec_ipv4->type == IBV_EXP_FLOW_SPEC_IPV4_EXT);
324		assert(spec_ipv4->size == sizeof(*spec_ipv4));
325
326		spec_ipv4->val.src_ip =
327			desc->src_ip[0] & mask->ipv4_mask.src_ip;
328		spec_ipv4->val.dst_ip =
329			desc->dst_ip[0] & mask->ipv4_mask.dst_ip;
330		spec_ipv4->mask.src_ip = mask->ipv4_mask.src_ip;
331		spec_ipv4->mask.dst_ip = mask->ipv4_mask.dst_ip;
332
333        /* PROTO */
334        spec_ipv4->val.proto  = desc->proto & mask->ipv4_mask.proto;
335        spec_ipv4->mask.proto = mask->ipv4_mask.proto;
336
337        /* TOS */
338        if (desc->ip_id ==1 ){
339            spec_ipv4->mask.tos = 0x1;
340        }else{
341            spec_ipv4->mask.tos = 0x0;
342        }
343        spec_ipv4->val.tos =
344                desc->tos & spec_ipv4->mask.tos;// & mask->ipv4_mask.tos;
345
346		/* Update priority */
347		attr->priority = 1;
348
349		if (desc->type == HASH_RXQ_IPV4)
350			goto create_flow;
351
352		spec_offset += spec_ipv4->size;
353		break;
354	case HASH_RXQ_IPV6:
355	case HASH_RXQ_UDPV6:
356	case HASH_RXQ_TCPV6:
357		spec_offset += spec_eth->size;
358
359		/* Set IP spec */
360		spec_ipv6 = (struct ibv_exp_flow_spec_ipv6_ext *)spec_offset;
361
362		/* The second specification must be IP. */
363		assert(spec_ipv6->type == IBV_EXP_FLOW_SPEC_IPV6);
364		assert(spec_ipv6->size == sizeof(*spec_ipv6));
365
366		for (i = 0; i != RTE_DIM(desc->src_ip); ++i) {
367			((uint32_t *)spec_ipv6->val.src_ip)[i] =
368				desc->src_ip[i] & mask->ipv6_mask.src_ip[i];
369			((uint32_t *)spec_ipv6->val.dst_ip)[i] =
370				desc->dst_ip[i] & mask->ipv6_mask.dst_ip[i];
371		}
372		rte_memcpy(spec_ipv6->mask.src_ip,
373			   mask->ipv6_mask.src_ip,
374			   sizeof(spec_ipv6->mask.src_ip));
375		rte_memcpy(spec_ipv6->mask.dst_ip,
376			   mask->ipv6_mask.dst_ip,
377			   sizeof(spec_ipv6->mask.dst_ip));
378
379		/* Update priority */
380		attr->priority = 1;
381
382		if (desc->type == HASH_RXQ_IPV6)
383			goto create_flow;
384
385		spec_offset += spec_ipv6->size;
386		break;
387	default:
388		ERROR("invalid flow attribute type");
389		return EINVAL;
390	}
391
392	/* Set TCP/UDP flow specification. */
393	spec_tcp_udp = (struct ibv_exp_flow_spec_tcp_udp *)spec_offset;
394
395	/* The third specification must be TCP/UDP. */
396	assert(spec_tcp_udp->type == IBV_EXP_FLOW_SPEC_TCP ||
397	       spec_tcp_udp->type == IBV_EXP_FLOW_SPEC_UDP);
398	assert(spec_tcp_udp->size == sizeof(*spec_tcp_udp));
399
400	spec_tcp_udp->val.src_port = desc->src_port & mask->src_port_mask;
401	spec_tcp_udp->val.dst_port = desc->dst_port & mask->dst_port_mask;
402	spec_tcp_udp->mask.src_port = mask->src_port_mask;
403	spec_tcp_udp->mask.dst_port = mask->dst_port_mask;
404
405	/* Update priority */
406	attr->priority = 0;
407
408create_flow:
409
410	errno = 0;
411	flow = ibv_exp_create_flow(fdir_queue->qp, attr);
412	if (flow == NULL) {
413		/* It's not clear whether errno is always set in this case. */
414		ERROR("%p: flow director configuration failed, errno=%d: %s",
415		      (void *)priv, errno,
416		      (errno ? strerror(errno) : "Unknown error"));
417		if (errno)
418			return errno;
419		return EINVAL;
420	}
421
422	DEBUG("%p: added flow director rule (%p)", (void *)priv, (void *)flow);
423	mlx5_fdir_filter->flow = flow;
424	return 0;
425}
426
427/**
428 * Get flow director queue for a specific RX queue, create it in case
429 * it does not exist.
430 *
431 * @param priv
432 *   Private structure.
433 * @param idx
434 *   RX queue index.
435 *
436 * @return
437 *   Related flow director queue on success, NULL otherwise.
438 */
439static struct fdir_queue *
440priv_get_fdir_queue(struct priv *priv, uint16_t idx)
441{
442	struct rxq_ctrl *rxq_ctrl =
443		container_of((*priv->rxqs)[idx], struct rxq_ctrl, rxq);
444	struct fdir_queue *fdir_queue = &rxq_ctrl->fdir_queue;
445	struct ibv_exp_rwq_ind_table *ind_table = NULL;
446	struct ibv_qp *qp = NULL;
447	struct ibv_exp_rwq_ind_table_init_attr ind_init_attr;
448	struct ibv_exp_rx_hash_conf hash_conf;
449	struct ibv_exp_qp_init_attr qp_init_attr;
450	int err = 0;
451
452	/* Return immediately if it has already been created. */
453	if (fdir_queue->qp != NULL)
454		return fdir_queue;
455
456	ind_init_attr = (struct ibv_exp_rwq_ind_table_init_attr){
457		.pd = priv->pd,
458		.log_ind_tbl_size = 0,
459		.ind_tbl = &rxq_ctrl->wq,
460		.comp_mask = 0,
461	};
462
463	errno = 0;
464	ind_table = ibv_exp_create_rwq_ind_table(priv->ctx,
465						 &ind_init_attr);
466	if (ind_table == NULL) {
467		/* Not clear whether errno is set. */
468		err = (errno ? errno : EINVAL);
469		ERROR("RX indirection table creation failed with error %d: %s",
470		      err, strerror(err));
471		goto error;
472	}
473
474	/* Create fdir_queue qp. */
475	hash_conf = (struct ibv_exp_rx_hash_conf){
476		.rx_hash_function = IBV_EXP_RX_HASH_FUNC_TOEPLITZ,
477		.rx_hash_key_len = rss_hash_default_key_len,
478		.rx_hash_key = rss_hash_default_key,
479		.rx_hash_fields_mask = 0,
480		.rwq_ind_tbl = ind_table,
481	};
482	qp_init_attr = (struct ibv_exp_qp_init_attr){
483		.max_inl_recv = 0, /* Currently not supported. */
484		.qp_type = IBV_QPT_RAW_PACKET,
485		.comp_mask = (IBV_EXP_QP_INIT_ATTR_PD |
486			      IBV_EXP_QP_INIT_ATTR_RX_HASH),
487		.pd = priv->pd,
488		.rx_hash_conf = &hash_conf,
489		.port_num = priv->port,
490	};
491
492	qp = ibv_exp_create_qp(priv->ctx, &qp_init_attr);
493	if (qp == NULL) {
494		err = (errno ? errno : EINVAL);
495		ERROR("hash RX QP creation failure: %s", strerror(err));
496		goto error;
497	}
498
499	fdir_queue->ind_table = ind_table;
500	fdir_queue->qp = qp;
501
502	return fdir_queue;
503
504error:
505	if (qp != NULL)
506		claim_zero(ibv_destroy_qp(qp));
507
508	if (ind_table != NULL)
509		claim_zero(ibv_exp_destroy_rwq_ind_table(ind_table));
510
511	return NULL;
512}
513
514/**
515 * Enable flow director filter and create steering rules.
516 *
517 * @param priv
518 *   Private structure.
519 * @param mlx5_fdir_filter
520 *   Filter to create steering rule for.
521 *
522 * @return
523 *   0 on success, errno value on failure.
524 */
525static int
526priv_fdir_filter_enable(struct priv *priv,
527			struct mlx5_fdir_filter *mlx5_fdir_filter)
528{
529	struct fdir_queue *fdir_queue;
530
531	/* Check if flow already exists. */
532	if (mlx5_fdir_filter->flow != NULL)
533		return 0;
534
535	/* Get fdir_queue for specific queue. */
536	fdir_queue = priv_get_fdir_queue(priv, mlx5_fdir_filter->queue);
537
538	if (fdir_queue == NULL) {
539		ERROR("failed to create flow director rxq for queue %d",
540		      mlx5_fdir_filter->queue);
541		return EINVAL;
542	}
543
544	/* Create flow */
545	return priv_fdir_flow_add(priv, mlx5_fdir_filter, fdir_queue);
546}
547
548/**
549 * Initialize flow director filters list.
550 *
551 * @param priv
552 *   Private structure.
553 *
554 * @return
555 *   0 on success, errno value on failure.
556 */
557int
558fdir_init_filters_list(struct priv *priv)
559{
560	/* Filter list initialization should be done only once. */
561	if (priv->fdir_filter_list)
562		return 0;
563
564	/* Create filters list. */
565	priv->fdir_filter_list =
566		rte_calloc(__func__, 1, sizeof(*priv->fdir_filter_list), 0);
567
568	if (priv->fdir_filter_list == NULL) {
569		int err = ENOMEM;
570
571		ERROR("cannot allocate flow director filter list: %s",
572		      strerror(err));
573		return err;
574	}
575
576	LIST_INIT(priv->fdir_filter_list);
577
578	return 0;
579}
580
581/**
582 * Flush all filters.
583 *
584 * @param priv
585 *   Private structure.
586 */
587static void
588priv_fdir_filter_flush(struct priv *priv)
589{
590	struct mlx5_fdir_filter *mlx5_fdir_filter;
591
592	while ((mlx5_fdir_filter = LIST_FIRST(priv->fdir_filter_list))) {
593		struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
594
595		DEBUG("%p: flushing flow director filter %p",
596		      (void *)priv, (void *)mlx5_fdir_filter);
597		LIST_REMOVE(mlx5_fdir_filter, next);
598		if (flow != NULL)
599			claim_zero(ibv_exp_destroy_flow(flow));
600		rte_free(mlx5_fdir_filter);
601	}
602}
603
604/**
605 * Remove all flow director filters and delete list.
606 *
607 * @param priv
608 *   Private structure.
609 */
610void
611priv_fdir_delete_filters_list(struct priv *priv)
612{
613	priv_fdir_filter_flush(priv);
614	rte_free(priv->fdir_filter_list);
615	priv->fdir_filter_list = NULL;
616}
617
618/**
619 * Disable flow director, remove all steering rules.
620 *
621 * @param priv
622 *   Private structure.
623 */
624void
625priv_fdir_disable(struct priv *priv)
626{
627	unsigned int i;
628	struct mlx5_fdir_filter *mlx5_fdir_filter;
629	struct fdir_queue *fdir_queue;
630
631	/* Run on every flow director filter and destroy flow handle. */
632	LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
633		struct ibv_exp_flow *flow;
634
635		/* Only valid elements should be in the list */
636		assert(mlx5_fdir_filter != NULL);
637		flow = mlx5_fdir_filter->flow;
638
639		/* Destroy flow handle */
640		if (flow != NULL) {
641			claim_zero(ibv_exp_destroy_flow(flow));
642			mlx5_fdir_filter->flow = NULL;
643		}
644	}
645
646	/* Run on every RX queue to destroy related flow director QP and
647	 * indirection table. */
648	for (i = 0; (i != priv->rxqs_n); i++) {
649		struct rxq_ctrl *rxq_ctrl =
650			container_of((*priv->rxqs)[i], struct rxq_ctrl, rxq);
651
652		fdir_queue = &rxq_ctrl->fdir_queue;
653		if (fdir_queue->qp != NULL) {
654			claim_zero(ibv_destroy_qp(fdir_queue->qp));
655			fdir_queue->qp = NULL;
656		}
657
658		if (fdir_queue->ind_table != NULL) {
659			claim_zero(ibv_exp_destroy_rwq_ind_table
660				   (fdir_queue->ind_table));
661			fdir_queue->ind_table = NULL;
662		}
663	}
664}
665
666/**
667 * Enable flow director, create steering rules.
668 *
669 * @param priv
670 *   Private structure.
671 */
672void
673priv_fdir_enable(struct priv *priv)
674{
675	struct mlx5_fdir_filter *mlx5_fdir_filter;
676
677	/* Run on every fdir filter and create flow handle */
678	LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
679		/* Only valid elements should be in the list */
680		assert(mlx5_fdir_filter != NULL);
681
682		priv_fdir_filter_enable(priv, mlx5_fdir_filter);
683	}
684}
685
686/**
687 * Find specific filter in list.
688 *
689 * @param priv
690 *   Private structure.
691 * @param fdir_filter
692 *   Flow director filter to find.
693 *
694 * @return
695 *   Filter element if found, otherwise NULL.
696 */
697static struct mlx5_fdir_filter *
698priv_find_filter_in_list(struct priv *priv,
699			 const struct rte_eth_fdir_filter *fdir_filter)
700{
701	struct fdir_flow_desc desc;
702	struct mlx5_fdir_filter *mlx5_fdir_filter;
703	enum rte_fdir_mode fdir_mode = priv->dev->data->dev_conf.fdir_conf.mode;
704
705	/* Get flow director filter to look for. */
706	fdir_filter_to_flow_desc(fdir_filter, &desc, fdir_mode);
707
708	/* Look for the requested element. */
709	LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
710		/* Only valid elements should be in the list. */
711		assert(mlx5_fdir_filter != NULL);
712
713		/* Return matching filter. */
714		if (!memcmp(&desc, &mlx5_fdir_filter->desc, sizeof(desc)))
715			return mlx5_fdir_filter;
716	}
717
718	/* Filter not found */
719	return NULL;
720}
721
722/**
723 * Add new flow director filter and store it in list.
724 *
725 * @param priv
726 *   Private structure.
727 * @param fdir_filter
728 *   Flow director filter to add.
729 *
730 * @return
731 *   0 on success, errno value on failure.
732 */
733static int
734priv_fdir_filter_add(struct priv *priv,
735		     const struct rte_eth_fdir_filter *fdir_filter)
736{
737	struct mlx5_fdir_filter *mlx5_fdir_filter;
738	enum rte_fdir_mode fdir_mode = priv->dev->data->dev_conf.fdir_conf.mode;
739	int err = 0;
740
741	/* Validate queue number. */
742	if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
743		ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
744		return EINVAL;
745	}
746
747	/* Duplicate filters are currently unsupported. */
748	mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
749	if (mlx5_fdir_filter != NULL) {
750		ERROR("filter already exists");
751		return EINVAL;
752	}
753
754	/* Create new flow director filter. */
755	mlx5_fdir_filter =
756		rte_calloc(__func__, 1, sizeof(*mlx5_fdir_filter), 0);
757	if (mlx5_fdir_filter == NULL) {
758		err = ENOMEM;
759		ERROR("cannot allocate flow director filter: %s",
760		      strerror(err));
761		return err;
762	}
763
764	/* Set queue. */
765	mlx5_fdir_filter->queue = fdir_filter->action.rx_queue;
766
767	/* Convert to mlx5 filter descriptor. */
768	fdir_filter_to_flow_desc(fdir_filter,
769				 &mlx5_fdir_filter->desc, fdir_mode);
770
771	/* Insert new filter into list. */
772	LIST_INSERT_HEAD(priv->fdir_filter_list, mlx5_fdir_filter, next);
773
774	DEBUG("%p: flow director filter %p added",
775	      (void *)priv, (void *)mlx5_fdir_filter);
776
777	/* Enable filter immediately if device is started. */
778	if (priv->started)
779		err = priv_fdir_filter_enable(priv, mlx5_fdir_filter);
780
781	return err;
782}
783
784/**
785 * Update queue for specific filter.
786 *
787 * @param priv
788 *   Private structure.
789 * @param fdir_filter
790 *   Filter to be updated.
791 *
792 * @return
793 *   0 on success, errno value on failure.
794 */
795static int
796priv_fdir_filter_update(struct priv *priv,
797			const struct rte_eth_fdir_filter *fdir_filter)
798{
799	struct mlx5_fdir_filter *mlx5_fdir_filter;
800
801	/* Validate queue number. */
802	if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
803		ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
804		return EINVAL;
805	}
806
807	mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
808	if (mlx5_fdir_filter != NULL) {
809		struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
810		int err = 0;
811
812		/* Update queue number. */
813		mlx5_fdir_filter->queue = fdir_filter->action.rx_queue;
814
815		/* Destroy flow handle. */
816		if (flow != NULL) {
817			claim_zero(ibv_exp_destroy_flow(flow));
818			mlx5_fdir_filter->flow = NULL;
819		}
820		DEBUG("%p: flow director filter %p updated",
821		      (void *)priv, (void *)mlx5_fdir_filter);
822
823		/* Enable filter if device is started. */
824		if (priv->started)
825			err = priv_fdir_filter_enable(priv, mlx5_fdir_filter);
826
827		return err;
828	}
829
830	/* Filter not found, create it. */
831	DEBUG("%p: filter not found for update, creating new filter",
832	      (void *)priv);
833	return priv_fdir_filter_add(priv, fdir_filter);
834}
835
836/**
837 * Delete specific filter.
838 *
839 * @param priv
840 *   Private structure.
841 * @param fdir_filter
842 *   Filter to be deleted.
843 *
844 * @return
845 *   0 on success, errno value on failure.
846 */
847static int
848priv_fdir_filter_delete(struct priv *priv,
849			const struct rte_eth_fdir_filter *fdir_filter)
850{
851	struct mlx5_fdir_filter *mlx5_fdir_filter;
852
853	mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
854	if (mlx5_fdir_filter != NULL) {
855		struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
856
857		/* Remove element from list. */
858		LIST_REMOVE(mlx5_fdir_filter, next);
859
860		/* Destroy flow handle. */
861		if (flow != NULL) {
862			claim_zero(ibv_exp_destroy_flow(flow));
863			mlx5_fdir_filter->flow = NULL;
864		}
865
866		DEBUG("%p: flow director filter %p deleted",
867		      (void *)priv, (void *)mlx5_fdir_filter);
868
869		/* Delete filter. */
870		rte_free(mlx5_fdir_filter);
871
872		return 0;
873	}
874
875	ERROR("%p: flow director delete failed, cannot find filter",
876	      (void *)priv);
877	return EINVAL;
878}
879
880/**
881 * Get flow director information.
882 *
883 * @param priv
884 *   Private structure.
885 * @param[out] fdir_info
886 *   Resulting flow director information.
887 */
888static void
889priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info)
890{
891	struct rte_eth_fdir_masks *mask =
892		&priv->dev->data->dev_conf.fdir_conf.mask;
893
894	fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode;
895	fdir_info->guarant_spc = 0;
896
897	rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
898
899	fdir_info->max_flexpayload = 0;
900	fdir_info->flow_types_mask[0] = 0;
901
902	fdir_info->flex_payload_unit = 0;
903	fdir_info->max_flex_payload_segment_num = 0;
904	fdir_info->flex_payload_limit = 0;
905	memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
906}
907
908/**
909 * Deal with flow director operations.
910 *
911 * @param priv
912 *   Pointer to private structure.
913 * @param filter_op
914 *   Operation to perform.
915 * @param arg
916 *   Pointer to operation-specific structure.
917 *
918 * @return
919 *   0 on success, errno value on failure.
920 */
921static int
922priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg)
923{
924	enum rte_fdir_mode fdir_mode =
925		priv->dev->data->dev_conf.fdir_conf.mode;
926	int ret = 0;
927
928	if (filter_op == RTE_ETH_FILTER_NOP)
929		return 0;
930
931	if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
932	    fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
933		ERROR("%p: flow director mode %d not supported",
934		      (void *)priv, fdir_mode);
935		return EINVAL;
936	}
937
938	switch (filter_op) {
939	case RTE_ETH_FILTER_ADD:
940		ret = priv_fdir_filter_add(priv, arg);
941		break;
942	case RTE_ETH_FILTER_UPDATE:
943		ret = priv_fdir_filter_update(priv, arg);
944		break;
945	case RTE_ETH_FILTER_DELETE:
946		ret = priv_fdir_filter_delete(priv, arg);
947		break;
948	case RTE_ETH_FILTER_FLUSH:
949		priv_fdir_filter_flush(priv);
950		break;
951	case RTE_ETH_FILTER_INFO:
952		priv_fdir_info_get(priv, arg);
953		break;
954	default:
955		DEBUG("%p: unknown operation %u", (void *)priv, filter_op);
956		ret = EINVAL;
957		break;
958	}
959	return ret;
960}
961
962/**
963 * Manage filter operations.
964 *
965 * @param dev
966 *   Pointer to Ethernet device structure.
967 * @param filter_type
968 *   Filter type.
969 * @param filter_op
970 *   Operation to perform.
971 * @param arg
972 *   Pointer to operation-specific structure.
973 *
974 * @return
975 *   0 on success, negative errno value on failure.
976 */
977int
978mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
979		     enum rte_filter_type filter_type,
980		     enum rte_filter_op filter_op,
981		     void *arg)
982{
983	int ret = -EINVAL;
984	struct priv *priv = dev->data->dev_private;
985
986	switch (filter_type) {
987	case RTE_ETH_FILTER_FDIR:
988		priv_lock(priv);
989		ret = priv_fdir_ctrl_func(priv, filter_op, arg);
990		priv_unlock(priv);
991		break;
992	default:
993		ERROR("%p: filter type (%d) not supported",
994		      (void *)dev, filter_type);
995		break;
996	}
997
998	return ret;
999}
1000