1/*-
2 *   BSD LICENSE
3 *
4 *   Copyright 2015 6WIND S.A.
5 *   Copyright 2015 Mellanox.
6 *
7 *   Redistribution and use in source and binary forms, with or without
8 *   modification, are permitted provided that the following conditions
9 *   are met:
10 *
11 *     * Redistributions of source code must retain the above copyright
12 *       notice, this list of conditions and the following disclaimer.
13 *     * Redistributions in binary form must reproduce the above copyright
14 *       notice, this list of conditions and the following disclaimer in
15 *       the documentation and/or other materials provided with the
16 *       distribution.
17 *     * Neither the name of 6WIND S.A. nor the names of its
18 *       contributors may be used to endorse or promote products derived
19 *       from this software without specific prior written permission.
20 *
21 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34#include <stddef.h>
35#include <assert.h>
36#include <stdint.h>
37#include <string.h>
38#include <errno.h>
39
40/* Verbs header. */
41/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
42#ifdef PEDANTIC
43#pragma GCC diagnostic ignored "-Wpedantic"
44#endif
45#include <infiniband/verbs.h>
46#ifdef PEDANTIC
47#pragma GCC diagnostic error "-Wpedantic"
48#endif
49
50/* DPDK headers don't like -pedantic. */
51#ifdef PEDANTIC
52#pragma GCC diagnostic ignored "-Wpedantic"
53#endif
54#include <rte_ether.h>
55#include <rte_malloc.h>
56#include <rte_ethdev.h>
57#include <rte_common.h>
58#ifdef PEDANTIC
59#pragma GCC diagnostic error "-Wpedantic"
60#endif
61
62#include "mlx5.h"
63#include "mlx5_rxtx.h"
64
65struct fdir_flow_desc {
66	uint16_t dst_port;
67	uint16_t src_port;
68	uint32_t src_ip[4];
69	uint32_t dst_ip[4];
70	uint8_t	mac[6];
71	uint16_t vlan_tag;
72	enum hash_rxq_type type;
73};
74
75struct mlx5_fdir_filter {
76	LIST_ENTRY(mlx5_fdir_filter) next;
77	uint16_t queue; /* Queue assigned to if FDIR match. */
78	enum rte_eth_fdir_behavior behavior;
79	struct fdir_flow_desc desc;
80	struct ibv_exp_flow *flow;
81};
82
83LIST_HEAD(fdir_filter_list, mlx5_fdir_filter);
84
85/**
86 * Convert struct rte_eth_fdir_filter to mlx5 filter descriptor.
87 *
88 * @param[in] fdir_filter
89 *   DPDK filter structure to convert.
90 * @param[out] desc
91 *   Resulting mlx5 filter descriptor.
92 * @param mode
93 *   Flow director mode.
94 */
95static void
96fdir_filter_to_flow_desc(const struct rte_eth_fdir_filter *fdir_filter,
97			 struct fdir_flow_desc *desc, enum rte_fdir_mode mode)
98{
99	/* Initialize descriptor. */
100	memset(desc, 0, sizeof(*desc));
101
102	/* Set VLAN ID. */
103	desc->vlan_tag = fdir_filter->input.flow_ext.vlan_tci;
104
105	/* Set MAC address. */
106	if (mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
107		rte_memcpy(desc->mac,
108			   fdir_filter->input.flow.mac_vlan_flow.mac_addr.
109				addr_bytes,
110			   sizeof(desc->mac));
111		desc->type = HASH_RXQ_ETH;
112		return;
113	}
114
115	/* Set mode */
116	switch (fdir_filter->input.flow_type) {
117	case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
118		desc->type = HASH_RXQ_UDPV4;
119		break;
120	case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
121		desc->type = HASH_RXQ_TCPV4;
122		break;
123	case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
124		desc->type = HASH_RXQ_IPV4;
125		break;
126	case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
127		desc->type = HASH_RXQ_UDPV6;
128		break;
129	case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
130		desc->type = HASH_RXQ_TCPV6;
131		break;
132	case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
133		desc->type = HASH_RXQ_IPV6;
134		break;
135	default:
136		break;
137	}
138
139	/* Set flow values */
140	switch (fdir_filter->input.flow_type) {
141	case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
142	case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
143		desc->src_port = fdir_filter->input.flow.udp4_flow.src_port;
144		desc->dst_port = fdir_filter->input.flow.udp4_flow.dst_port;
145		/* fallthrough */
146	case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
147		desc->src_ip[0] = fdir_filter->input.flow.ip4_flow.src_ip;
148		desc->dst_ip[0] = fdir_filter->input.flow.ip4_flow.dst_ip;
149		break;
150	case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
151	case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
152		desc->src_port = fdir_filter->input.flow.udp6_flow.src_port;
153		desc->dst_port = fdir_filter->input.flow.udp6_flow.dst_port;
154		/* Fall through. */
155	case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
156		rte_memcpy(desc->src_ip,
157			   fdir_filter->input.flow.ipv6_flow.src_ip,
158			   sizeof(desc->src_ip));
159		rte_memcpy(desc->dst_ip,
160			   fdir_filter->input.flow.ipv6_flow.dst_ip,
161			   sizeof(desc->dst_ip));
162		break;
163	default:
164		break;
165	}
166}
167
168/**
169 * Check if two flow descriptors overlap according to configured mask.
170 *
171 * @param priv
172 *   Private structure that provides flow director mask.
173 * @param desc1
174 *   First flow descriptor to compare.
175 * @param desc2
176 *   Second flow descriptor to compare.
177 *
178 * @return
179 *   Nonzero if descriptors overlap.
180 */
181static int
182priv_fdir_overlap(const struct priv *priv,
183		  const struct fdir_flow_desc *desc1,
184		  const struct fdir_flow_desc *desc2)
185{
186	const struct rte_eth_fdir_masks *mask =
187		&priv->dev->data->dev_conf.fdir_conf.mask;
188	unsigned int i;
189
190	if (desc1->type != desc2->type)
191		return 0;
192	/* Ignore non masked bits. */
193	for (i = 0; i != RTE_DIM(desc1->mac); ++i)
194		if ((desc1->mac[i] & mask->mac_addr_byte_mask) !=
195		    (desc2->mac[i] & mask->mac_addr_byte_mask))
196			return 0;
197	if (((desc1->src_port & mask->src_port_mask) !=
198	     (desc2->src_port & mask->src_port_mask)) ||
199	    ((desc1->dst_port & mask->dst_port_mask) !=
200	     (desc2->dst_port & mask->dst_port_mask)))
201		return 0;
202	switch (desc1->type) {
203	case HASH_RXQ_IPV4:
204	case HASH_RXQ_UDPV4:
205	case HASH_RXQ_TCPV4:
206		if (((desc1->src_ip[0] & mask->ipv4_mask.src_ip) !=
207		     (desc2->src_ip[0] & mask->ipv4_mask.src_ip)) ||
208		    ((desc1->dst_ip[0] & mask->ipv4_mask.dst_ip) !=
209		     (desc2->dst_ip[0] & mask->ipv4_mask.dst_ip)))
210			return 0;
211		break;
212	case HASH_RXQ_IPV6:
213	case HASH_RXQ_UDPV6:
214	case HASH_RXQ_TCPV6:
215		for (i = 0; i != RTE_DIM(desc1->src_ip); ++i)
216			if (((desc1->src_ip[i] & mask->ipv6_mask.src_ip[i]) !=
217			     (desc2->src_ip[i] & mask->ipv6_mask.src_ip[i])) ||
218			    ((desc1->dst_ip[i] & mask->ipv6_mask.dst_ip[i]) !=
219			     (desc2->dst_ip[i] & mask->ipv6_mask.dst_ip[i])))
220				return 0;
221		break;
222	default:
223		break;
224	}
225	return 1;
226}
227
228/**
229 * Create flow director steering rule for a specific filter.
230 *
231 * @param priv
232 *   Private structure.
233 * @param mlx5_fdir_filter
234 *   Filter to create a steering rule for.
235 * @param fdir_queue
236 *   Flow director queue for matching packets.
237 *
238 * @return
239 *   0 on success, errno value on failure.
240 */
241static int
242priv_fdir_flow_add(struct priv *priv,
243		   struct mlx5_fdir_filter *mlx5_fdir_filter,
244		   struct fdir_queue *fdir_queue)
245{
246	struct ibv_exp_flow *flow;
247	struct fdir_flow_desc *desc = &mlx5_fdir_filter->desc;
248	enum rte_fdir_mode fdir_mode =
249		priv->dev->data->dev_conf.fdir_conf.mode;
250	struct rte_eth_fdir_masks *mask =
251		&priv->dev->data->dev_conf.fdir_conf.mask;
252	FLOW_ATTR_SPEC_ETH(data, priv_flow_attr(priv, NULL, 0, desc->type));
253	struct ibv_exp_flow_attr *attr = &data->attr;
254	uintptr_t spec_offset = (uintptr_t)&data->spec;
255	struct ibv_exp_flow_spec_eth *spec_eth;
256	struct ibv_exp_flow_spec_ipv4 *spec_ipv4;
257	struct ibv_exp_flow_spec_ipv6 *spec_ipv6;
258	struct ibv_exp_flow_spec_tcp_udp *spec_tcp_udp;
259	struct mlx5_fdir_filter *iter_fdir_filter;
260	unsigned int i;
261
262	/* Abort if an existing flow overlaps this one to avoid packet
263	 * duplication, even if it targets another queue. */
264	LIST_FOREACH(iter_fdir_filter, priv->fdir_filter_list, next)
265		if ((iter_fdir_filter != mlx5_fdir_filter) &&
266		    (iter_fdir_filter->flow != NULL) &&
267		    (priv_fdir_overlap(priv,
268				       &mlx5_fdir_filter->desc,
269				       &iter_fdir_filter->desc)))
270			return EEXIST;
271
272	/*
273	 * No padding must be inserted by the compiler between attr and spec.
274	 * This layout is expected by libibverbs.
275	 */
276	assert(((uint8_t *)attr + sizeof(*attr)) == (uint8_t *)spec_offset);
277	priv_flow_attr(priv, attr, sizeof(data), desc->type);
278
279	/* Set Ethernet spec */
280	spec_eth = (struct ibv_exp_flow_spec_eth *)spec_offset;
281
282	/* The first specification must be Ethernet. */
283	assert(spec_eth->type == IBV_EXP_FLOW_SPEC_ETH);
284	assert(spec_eth->size == sizeof(*spec_eth));
285
286	/* VLAN ID */
287	spec_eth->val.vlan_tag = desc->vlan_tag & mask->vlan_tci_mask;
288	spec_eth->mask.vlan_tag = mask->vlan_tci_mask;
289
290	/* Update priority */
291	attr->priority = 2;
292
293	if (fdir_mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
294		/* MAC Address */
295		for (i = 0; i != RTE_DIM(spec_eth->mask.dst_mac); ++i) {
296			spec_eth->val.dst_mac[i] =
297				desc->mac[i] & mask->mac_addr_byte_mask;
298			spec_eth->mask.dst_mac[i] = mask->mac_addr_byte_mask;
299		}
300		goto create_flow;
301	}
302
303	switch (desc->type) {
304	case HASH_RXQ_IPV4:
305	case HASH_RXQ_UDPV4:
306	case HASH_RXQ_TCPV4:
307		spec_offset += spec_eth->size;
308
309		/* Set IP spec */
310		spec_ipv4 = (struct ibv_exp_flow_spec_ipv4 *)spec_offset;
311
312		/* The second specification must be IP. */
313		assert(spec_ipv4->type == IBV_EXP_FLOW_SPEC_IPV4);
314		assert(spec_ipv4->size == sizeof(*spec_ipv4));
315
316		spec_ipv4->val.src_ip =
317			desc->src_ip[0] & mask->ipv4_mask.src_ip;
318		spec_ipv4->val.dst_ip =
319			desc->dst_ip[0] & mask->ipv4_mask.dst_ip;
320		spec_ipv4->mask.src_ip = mask->ipv4_mask.src_ip;
321		spec_ipv4->mask.dst_ip = mask->ipv4_mask.dst_ip;
322
323		/* Update priority */
324		attr->priority = 1;
325
326		if (desc->type == HASH_RXQ_IPV4)
327			goto create_flow;
328
329		spec_offset += spec_ipv4->size;
330		break;
331	case HASH_RXQ_IPV6:
332	case HASH_RXQ_UDPV6:
333	case HASH_RXQ_TCPV6:
334		spec_offset += spec_eth->size;
335
336		/* Set IP spec */
337		spec_ipv6 = (struct ibv_exp_flow_spec_ipv6 *)spec_offset;
338
339		/* The second specification must be IP. */
340		assert(spec_ipv6->type == IBV_EXP_FLOW_SPEC_IPV6);
341		assert(spec_ipv6->size == sizeof(*spec_ipv6));
342
343		for (i = 0; i != RTE_DIM(desc->src_ip); ++i) {
344			((uint32_t *)spec_ipv6->val.src_ip)[i] =
345				desc->src_ip[i] & mask->ipv6_mask.src_ip[i];
346			((uint32_t *)spec_ipv6->val.dst_ip)[i] =
347				desc->dst_ip[i] & mask->ipv6_mask.dst_ip[i];
348		}
349		rte_memcpy(spec_ipv6->mask.src_ip,
350			   mask->ipv6_mask.src_ip,
351			   sizeof(spec_ipv6->mask.src_ip));
352		rte_memcpy(spec_ipv6->mask.dst_ip,
353			   mask->ipv6_mask.dst_ip,
354			   sizeof(spec_ipv6->mask.dst_ip));
355
356		/* Update priority */
357		attr->priority = 1;
358
359		if (desc->type == HASH_RXQ_IPV6)
360			goto create_flow;
361
362		spec_offset += spec_ipv6->size;
363		break;
364	default:
365		ERROR("invalid flow attribute type");
366		return EINVAL;
367	}
368
369	/* Set TCP/UDP flow specification. */
370	spec_tcp_udp = (struct ibv_exp_flow_spec_tcp_udp *)spec_offset;
371
372	/* The third specification must be TCP/UDP. */
373	assert(spec_tcp_udp->type == IBV_EXP_FLOW_SPEC_TCP ||
374	       spec_tcp_udp->type == IBV_EXP_FLOW_SPEC_UDP);
375	assert(spec_tcp_udp->size == sizeof(*spec_tcp_udp));
376
377	spec_tcp_udp->val.src_port = desc->src_port & mask->src_port_mask;
378	spec_tcp_udp->val.dst_port = desc->dst_port & mask->dst_port_mask;
379	spec_tcp_udp->mask.src_port = mask->src_port_mask;
380	spec_tcp_udp->mask.dst_port = mask->dst_port_mask;
381
382	/* Update priority */
383	attr->priority = 0;
384
385create_flow:
386
387	errno = 0;
388	flow = ibv_exp_create_flow(fdir_queue->qp, attr);
389	if (flow == NULL) {
390		/* It's not clear whether errno is always set in this case. */
391		ERROR("%p: flow director configuration failed, errno=%d: %s",
392		      (void *)priv, errno,
393		      (errno ? strerror(errno) : "Unknown error"));
394		if (errno)
395			return errno;
396		return EINVAL;
397	}
398
399	DEBUG("%p: added flow director rule (%p)", (void *)priv, (void *)flow);
400	mlx5_fdir_filter->flow = flow;
401	return 0;
402}
403
404/**
405 * Destroy a flow director queue.
406 *
407 * @param fdir_queue
408 *   Flow director queue to be destroyed.
409 */
410void
411priv_fdir_queue_destroy(struct priv *priv, struct fdir_queue *fdir_queue)
412{
413	struct mlx5_fdir_filter *fdir_filter;
414
415	/* Disable filter flows still applying to this queue. */
416	LIST_FOREACH(fdir_filter, priv->fdir_filter_list, next) {
417		unsigned int idx = fdir_filter->queue;
418		struct rxq_ctrl *rxq_ctrl =
419			container_of((*priv->rxqs)[idx], struct rxq_ctrl, rxq);
420
421		assert(idx < priv->rxqs_n);
422		if (fdir_queue == rxq_ctrl->fdir_queue &&
423		    fdir_filter->flow != NULL) {
424			claim_zero(ibv_exp_destroy_flow(fdir_filter->flow));
425			fdir_filter->flow = NULL;
426		}
427	}
428	assert(fdir_queue->qp);
429	claim_zero(ibv_destroy_qp(fdir_queue->qp));
430	assert(fdir_queue->ind_table);
431	claim_zero(ibv_exp_destroy_rwq_ind_table(fdir_queue->ind_table));
432	if (fdir_queue->wq)
433		claim_zero(ibv_exp_destroy_wq(fdir_queue->wq));
434	if (fdir_queue->cq)
435		claim_zero(ibv_destroy_cq(fdir_queue->cq));
436#ifndef NDEBUG
437	memset(fdir_queue, 0x2a, sizeof(*fdir_queue));
438#endif
439	rte_free(fdir_queue);
440}
441
442/**
443 * Create a flow director queue.
444 *
445 * @param priv
446 *   Private structure.
447 * @param wq
448 *   Work queue to route matched packets to, NULL if one needs to
449 *   be created.
450 *
451 * @return
452 *   Related flow director queue on success, NULL otherwise.
453 */
454static struct fdir_queue *
455priv_fdir_queue_create(struct priv *priv, struct ibv_exp_wq *wq,
456		       unsigned int socket)
457{
458	struct fdir_queue *fdir_queue;
459
460	fdir_queue = rte_calloc_socket(__func__, 1, sizeof(*fdir_queue),
461				       0, socket);
462	if (!fdir_queue) {
463		ERROR("cannot allocate flow director queue");
464		return NULL;
465	}
466	assert(priv->pd);
467	assert(priv->ctx);
468	if (!wq) {
469		fdir_queue->cq = ibv_exp_create_cq(
470			priv->ctx, 1, NULL, NULL, 0,
471			&(struct ibv_exp_cq_init_attr){
472				.comp_mask = 0,
473			});
474		if (!fdir_queue->cq) {
475			ERROR("cannot create flow director CQ");
476			goto error;
477		}
478		fdir_queue->wq = ibv_exp_create_wq(
479			priv->ctx,
480			&(struct ibv_exp_wq_init_attr){
481				.wq_type = IBV_EXP_WQT_RQ,
482				.max_recv_wr = 1,
483				.max_recv_sge = 1,
484				.pd = priv->pd,
485				.cq = fdir_queue->cq,
486			});
487		if (!fdir_queue->wq) {
488			ERROR("cannot create flow director WQ");
489			goto error;
490		}
491		wq = fdir_queue->wq;
492	}
493	fdir_queue->ind_table = ibv_exp_create_rwq_ind_table(
494		priv->ctx,
495		&(struct ibv_exp_rwq_ind_table_init_attr){
496			.pd = priv->pd,
497			.log_ind_tbl_size = 0,
498			.ind_tbl = &wq,
499			.comp_mask = 0,
500		});
501	if (!fdir_queue->ind_table) {
502		ERROR("cannot create flow director indirection table");
503		goto error;
504	}
505	fdir_queue->qp = ibv_exp_create_qp(
506		priv->ctx,
507		&(struct ibv_exp_qp_init_attr){
508			.qp_type = IBV_QPT_RAW_PACKET,
509			.comp_mask =
510				IBV_EXP_QP_INIT_ATTR_PD |
511				IBV_EXP_QP_INIT_ATTR_PORT |
512				IBV_EXP_QP_INIT_ATTR_RX_HASH,
513			.pd = priv->pd,
514			.rx_hash_conf = &(struct ibv_exp_rx_hash_conf){
515				.rx_hash_function =
516					IBV_EXP_RX_HASH_FUNC_TOEPLITZ,
517				.rx_hash_key_len = rss_hash_default_key_len,
518				.rx_hash_key = rss_hash_default_key,
519				.rx_hash_fields_mask = 0,
520				.rwq_ind_tbl = fdir_queue->ind_table,
521			},
522			.port_num = priv->port,
523		});
524	if (!fdir_queue->qp) {
525		ERROR("cannot create flow director hash RX QP");
526		goto error;
527	}
528	return fdir_queue;
529error:
530	assert(fdir_queue);
531	assert(!fdir_queue->qp);
532	if (fdir_queue->ind_table)
533		claim_zero(ibv_exp_destroy_rwq_ind_table
534			   (fdir_queue->ind_table));
535	if (fdir_queue->wq)
536		claim_zero(ibv_exp_destroy_wq(fdir_queue->wq));
537	if (fdir_queue->cq)
538		claim_zero(ibv_destroy_cq(fdir_queue->cq));
539	rte_free(fdir_queue);
540	return NULL;
541}
542
543/**
544 * Get flow director queue for a specific RX queue, create it in case
545 * it does not exist.
546 *
547 * @param priv
548 *   Private structure.
549 * @param idx
550 *   RX queue index.
551 *
552 * @return
553 *   Related flow director queue on success, NULL otherwise.
554 */
555static struct fdir_queue *
556priv_get_fdir_queue(struct priv *priv, uint16_t idx)
557{
558	struct rxq_ctrl *rxq_ctrl =
559		container_of((*priv->rxqs)[idx], struct rxq_ctrl, rxq);
560	struct fdir_queue *fdir_queue = rxq_ctrl->fdir_queue;
561
562	assert(rxq_ctrl->wq);
563	if (fdir_queue == NULL) {
564		fdir_queue = priv_fdir_queue_create(priv, rxq_ctrl->wq,
565						    rxq_ctrl->socket);
566		rxq_ctrl->fdir_queue = fdir_queue;
567	}
568	return fdir_queue;
569}
570
571/**
572 * Get or flow director drop queue. Create it if it does not exist.
573 *
574 * @param priv
575 *   Private structure.
576 *
577 * @return
578 *   Flow director drop queue on success, NULL otherwise.
579 */
580static struct fdir_queue *
581priv_get_fdir_drop_queue(struct priv *priv)
582{
583	struct fdir_queue *fdir_queue = priv->fdir_drop_queue;
584
585	if (fdir_queue == NULL) {
586		unsigned int socket = SOCKET_ID_ANY;
587
588		/* Select a known NUMA socket if possible. */
589		if (priv->rxqs_n && (*priv->rxqs)[0])
590			socket = container_of((*priv->rxqs)[0],
591					      struct rxq_ctrl, rxq)->socket;
592		fdir_queue = priv_fdir_queue_create(priv, NULL, socket);
593		priv->fdir_drop_queue = fdir_queue;
594	}
595	return fdir_queue;
596}
597
598/**
599 * Enable flow director filter and create steering rules.
600 *
601 * @param priv
602 *   Private structure.
603 * @param mlx5_fdir_filter
604 *   Filter to create steering rule for.
605 *
606 * @return
607 *   0 on success, errno value on failure.
608 */
609static int
610priv_fdir_filter_enable(struct priv *priv,
611			struct mlx5_fdir_filter *mlx5_fdir_filter)
612{
613	struct fdir_queue *fdir_queue;
614
615	/* Check if flow already exists. */
616	if (mlx5_fdir_filter->flow != NULL)
617		return 0;
618
619	/* Get fdir_queue for specific queue. */
620	if (mlx5_fdir_filter->behavior == RTE_ETH_FDIR_REJECT)
621		fdir_queue = priv_get_fdir_drop_queue(priv);
622	else
623		fdir_queue = priv_get_fdir_queue(priv,
624						 mlx5_fdir_filter->queue);
625
626	if (fdir_queue == NULL) {
627		ERROR("failed to create flow director rxq for queue %d",
628		      mlx5_fdir_filter->queue);
629		return EINVAL;
630	}
631
632	/* Create flow */
633	return priv_fdir_flow_add(priv, mlx5_fdir_filter, fdir_queue);
634}
635
636/**
637 * Initialize flow director filters list.
638 *
639 * @param priv
640 *   Private structure.
641 *
642 * @return
643 *   0 on success, errno value on failure.
644 */
645int
646fdir_init_filters_list(struct priv *priv)
647{
648	/* Filter list initialization should be done only once. */
649	if (priv->fdir_filter_list)
650		return 0;
651
652	/* Create filters list. */
653	priv->fdir_filter_list =
654		rte_calloc(__func__, 1, sizeof(*priv->fdir_filter_list), 0);
655
656	if (priv->fdir_filter_list == NULL) {
657		int err = ENOMEM;
658
659		ERROR("cannot allocate flow director filter list: %s",
660		      strerror(err));
661		return err;
662	}
663
664	LIST_INIT(priv->fdir_filter_list);
665
666	return 0;
667}
668
669/**
670 * Flush all filters.
671 *
672 * @param priv
673 *   Private structure.
674 */
675static void
676priv_fdir_filter_flush(struct priv *priv)
677{
678	struct mlx5_fdir_filter *mlx5_fdir_filter;
679
680	while ((mlx5_fdir_filter = LIST_FIRST(priv->fdir_filter_list))) {
681		struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
682
683		DEBUG("%p: flushing flow director filter %p",
684		      (void *)priv, (void *)mlx5_fdir_filter);
685		LIST_REMOVE(mlx5_fdir_filter, next);
686		if (flow != NULL)
687			claim_zero(ibv_exp_destroy_flow(flow));
688		rte_free(mlx5_fdir_filter);
689	}
690}
691
692/**
693 * Remove all flow director filters and delete list.
694 *
695 * @param priv
696 *   Private structure.
697 */
698void
699priv_fdir_delete_filters_list(struct priv *priv)
700{
701	priv_fdir_filter_flush(priv);
702	rte_free(priv->fdir_filter_list);
703	priv->fdir_filter_list = NULL;
704}
705
706/**
707 * Disable flow director, remove all steering rules.
708 *
709 * @param priv
710 *   Private structure.
711 */
712void
713priv_fdir_disable(struct priv *priv)
714{
715	unsigned int i;
716	struct mlx5_fdir_filter *mlx5_fdir_filter;
717
718	/* Run on every flow director filter and destroy flow handle. */
719	LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
720		struct ibv_exp_flow *flow;
721
722		/* Only valid elements should be in the list */
723		assert(mlx5_fdir_filter != NULL);
724		flow = mlx5_fdir_filter->flow;
725
726		/* Destroy flow handle */
727		if (flow != NULL) {
728			claim_zero(ibv_exp_destroy_flow(flow));
729			mlx5_fdir_filter->flow = NULL;
730		}
731	}
732
733	/* Destroy flow director context in each RX queue. */
734	for (i = 0; (i != priv->rxqs_n); i++) {
735		struct rxq_ctrl *rxq_ctrl;
736
737		if (!(*priv->rxqs)[i])
738			continue;
739		rxq_ctrl = container_of((*priv->rxqs)[i], struct rxq_ctrl, rxq);
740		if (!rxq_ctrl->fdir_queue)
741			continue;
742		priv_fdir_queue_destroy(priv, rxq_ctrl->fdir_queue);
743		rxq_ctrl->fdir_queue = NULL;
744	}
745	if (priv->fdir_drop_queue) {
746		priv_fdir_queue_destroy(priv, priv->fdir_drop_queue);
747		priv->fdir_drop_queue = NULL;
748	}
749}
750
751/**
752 * Enable flow director, create steering rules.
753 *
754 * @param priv
755 *   Private structure.
756 */
757void
758priv_fdir_enable(struct priv *priv)
759{
760	struct mlx5_fdir_filter *mlx5_fdir_filter;
761
762	/* Run on every fdir filter and create flow handle */
763	LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
764		/* Only valid elements should be in the list */
765		assert(mlx5_fdir_filter != NULL);
766
767		priv_fdir_filter_enable(priv, mlx5_fdir_filter);
768	}
769}
770
771/**
772 * Find specific filter in list.
773 *
774 * @param priv
775 *   Private structure.
776 * @param fdir_filter
777 *   Flow director filter to find.
778 *
779 * @return
780 *   Filter element if found, otherwise NULL.
781 */
782static struct mlx5_fdir_filter *
783priv_find_filter_in_list(struct priv *priv,
784			 const struct rte_eth_fdir_filter *fdir_filter)
785{
786	struct fdir_flow_desc desc;
787	struct mlx5_fdir_filter *mlx5_fdir_filter;
788	enum rte_fdir_mode fdir_mode = priv->dev->data->dev_conf.fdir_conf.mode;
789
790	/* Get flow director filter to look for. */
791	fdir_filter_to_flow_desc(fdir_filter, &desc, fdir_mode);
792
793	/* Look for the requested element. */
794	LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
795		/* Only valid elements should be in the list. */
796		assert(mlx5_fdir_filter != NULL);
797
798		/* Return matching filter. */
799		if (!memcmp(&desc, &mlx5_fdir_filter->desc, sizeof(desc)))
800			return mlx5_fdir_filter;
801	}
802
803	/* Filter not found */
804	return NULL;
805}
806
807/**
808 * Add new flow director filter and store it in list.
809 *
810 * @param priv
811 *   Private structure.
812 * @param fdir_filter
813 *   Flow director filter to add.
814 *
815 * @return
816 *   0 on success, errno value on failure.
817 */
818static int
819priv_fdir_filter_add(struct priv *priv,
820		     const struct rte_eth_fdir_filter *fdir_filter)
821{
822	struct mlx5_fdir_filter *mlx5_fdir_filter;
823	enum rte_fdir_mode fdir_mode = priv->dev->data->dev_conf.fdir_conf.mode;
824	int err = 0;
825
826	/* Validate queue number. */
827	if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
828		ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
829		return EINVAL;
830	}
831
832	/* Duplicate filters are currently unsupported. */
833	mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
834	if (mlx5_fdir_filter != NULL) {
835		ERROR("filter already exists");
836		return EINVAL;
837	}
838
839	/* Create new flow director filter. */
840	mlx5_fdir_filter =
841		rte_calloc(__func__, 1, sizeof(*mlx5_fdir_filter), 0);
842	if (mlx5_fdir_filter == NULL) {
843		err = ENOMEM;
844		ERROR("cannot allocate flow director filter: %s",
845		      strerror(err));
846		return err;
847	}
848
849	/* Set action parameters. */
850	mlx5_fdir_filter->queue = fdir_filter->action.rx_queue;
851	mlx5_fdir_filter->behavior = fdir_filter->action.behavior;
852
853	/* Convert to mlx5 filter descriptor. */
854	fdir_filter_to_flow_desc(fdir_filter,
855				 &mlx5_fdir_filter->desc, fdir_mode);
856
857	/* Insert new filter into list. */
858	LIST_INSERT_HEAD(priv->fdir_filter_list, mlx5_fdir_filter, next);
859
860	DEBUG("%p: flow director filter %p added",
861	      (void *)priv, (void *)mlx5_fdir_filter);
862
863	/* Enable filter immediately if device is started. */
864	if (priv->started)
865		err = priv_fdir_filter_enable(priv, mlx5_fdir_filter);
866
867	return err;
868}
869
870/**
871 * Update queue for specific filter.
872 *
873 * @param priv
874 *   Private structure.
875 * @param fdir_filter
876 *   Filter to be updated.
877 *
878 * @return
879 *   0 on success, errno value on failure.
880 */
881static int
882priv_fdir_filter_update(struct priv *priv,
883			const struct rte_eth_fdir_filter *fdir_filter)
884{
885	struct mlx5_fdir_filter *mlx5_fdir_filter;
886
887	/* Validate queue number. */
888	if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
889		ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
890		return EINVAL;
891	}
892
893	mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
894	if (mlx5_fdir_filter != NULL) {
895		struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
896		int err = 0;
897
898		/* Update queue number. */
899		mlx5_fdir_filter->queue = fdir_filter->action.rx_queue;
900
901		/* Destroy flow handle. */
902		if (flow != NULL) {
903			claim_zero(ibv_exp_destroy_flow(flow));
904			mlx5_fdir_filter->flow = NULL;
905		}
906		DEBUG("%p: flow director filter %p updated",
907		      (void *)priv, (void *)mlx5_fdir_filter);
908
909		/* Enable filter if device is started. */
910		if (priv->started)
911			err = priv_fdir_filter_enable(priv, mlx5_fdir_filter);
912
913		return err;
914	}
915
916	/* Filter not found, create it. */
917	DEBUG("%p: filter not found for update, creating new filter",
918	      (void *)priv);
919	return priv_fdir_filter_add(priv, fdir_filter);
920}
921
922/**
923 * Delete specific filter.
924 *
925 * @param priv
926 *   Private structure.
927 * @param fdir_filter
928 *   Filter to be deleted.
929 *
930 * @return
931 *   0 on success, errno value on failure.
932 */
933static int
934priv_fdir_filter_delete(struct priv *priv,
935			const struct rte_eth_fdir_filter *fdir_filter)
936{
937	struct mlx5_fdir_filter *mlx5_fdir_filter;
938
939	mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
940	if (mlx5_fdir_filter != NULL) {
941		struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
942
943		/* Remove element from list. */
944		LIST_REMOVE(mlx5_fdir_filter, next);
945
946		/* Destroy flow handle. */
947		if (flow != NULL) {
948			claim_zero(ibv_exp_destroy_flow(flow));
949			mlx5_fdir_filter->flow = NULL;
950		}
951
952		DEBUG("%p: flow director filter %p deleted",
953		      (void *)priv, (void *)mlx5_fdir_filter);
954
955		/* Delete filter. */
956		rte_free(mlx5_fdir_filter);
957
958		return 0;
959	}
960
961	ERROR("%p: flow director delete failed, cannot find filter",
962	      (void *)priv);
963	return EINVAL;
964}
965
966/**
967 * Get flow director information.
968 *
969 * @param priv
970 *   Private structure.
971 * @param[out] fdir_info
972 *   Resulting flow director information.
973 */
974static void
975priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info)
976{
977	struct rte_eth_fdir_masks *mask =
978		&priv->dev->data->dev_conf.fdir_conf.mask;
979
980	fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode;
981	fdir_info->guarant_spc = 0;
982
983	rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
984
985	fdir_info->max_flexpayload = 0;
986	fdir_info->flow_types_mask[0] = 0;
987
988	fdir_info->flex_payload_unit = 0;
989	fdir_info->max_flex_payload_segment_num = 0;
990	fdir_info->flex_payload_limit = 0;
991	memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
992}
993
994/**
995 * Deal with flow director operations.
996 *
997 * @param priv
998 *   Pointer to private structure.
999 * @param filter_op
1000 *   Operation to perform.
1001 * @param arg
1002 *   Pointer to operation-specific structure.
1003 *
1004 * @return
1005 *   0 on success, errno value on failure.
1006 */
1007static int
1008priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg)
1009{
1010	enum rte_fdir_mode fdir_mode =
1011		priv->dev->data->dev_conf.fdir_conf.mode;
1012	int ret = 0;
1013
1014	if (filter_op == RTE_ETH_FILTER_NOP)
1015		return 0;
1016
1017	if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
1018	    fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
1019		ERROR("%p: flow director mode %d not supported",
1020		      (void *)priv, fdir_mode);
1021		return EINVAL;
1022	}
1023
1024	switch (filter_op) {
1025	case RTE_ETH_FILTER_ADD:
1026		ret = priv_fdir_filter_add(priv, arg);
1027		break;
1028	case RTE_ETH_FILTER_UPDATE:
1029		ret = priv_fdir_filter_update(priv, arg);
1030		break;
1031	case RTE_ETH_FILTER_DELETE:
1032		ret = priv_fdir_filter_delete(priv, arg);
1033		break;
1034	case RTE_ETH_FILTER_FLUSH:
1035		priv_fdir_filter_flush(priv);
1036		break;
1037	case RTE_ETH_FILTER_INFO:
1038		priv_fdir_info_get(priv, arg);
1039		break;
1040	default:
1041		DEBUG("%p: unknown operation %u", (void *)priv, filter_op);
1042		ret = EINVAL;
1043		break;
1044	}
1045	return ret;
1046}
1047
1048/**
1049 * Manage filter operations.
1050 *
1051 * @param dev
1052 *   Pointer to Ethernet device structure.
1053 * @param filter_type
1054 *   Filter type.
1055 * @param filter_op
1056 *   Operation to perform.
1057 * @param arg
1058 *   Pointer to operation-specific structure.
1059 *
1060 * @return
1061 *   0 on success, negative errno value on failure.
1062 */
1063int
1064mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
1065		     enum rte_filter_type filter_type,
1066		     enum rte_filter_op filter_op,
1067		     void *arg)
1068{
1069	int ret = EINVAL;
1070	struct priv *priv = dev->data->dev_private;
1071
1072	switch (filter_type) {
1073	case RTE_ETH_FILTER_FDIR:
1074		priv_lock(priv);
1075		ret = priv_fdir_ctrl_func(priv, filter_op, arg);
1076		priv_unlock(priv);
1077		break;
1078	default:
1079		ERROR("%p: filter type (%d) not supported",
1080		      (void *)dev, filter_type);
1081		break;
1082	}
1083
1084	return -ret;
1085}
1086