197f17497SC.J. Collier/*-
297f17497SC.J. Collier *   BSD LICENSE
397f17497SC.J. Collier *
497f17497SC.J. Collier *   Copyright 2015 6WIND S.A.
597f17497SC.J. Collier *   Copyright 2015 Mellanox.
697f17497SC.J. Collier *
797f17497SC.J. Collier *   Redistribution and use in source and binary forms, with or without
897f17497SC.J. Collier *   modification, are permitted provided that the following conditions
997f17497SC.J. Collier *   are met:
1097f17497SC.J. Collier *
1197f17497SC.J. Collier *     * Redistributions of source code must retain the above copyright
1297f17497SC.J. Collier *       notice, this list of conditions and the following disclaimer.
1397f17497SC.J. Collier *     * Redistributions in binary form must reproduce the above copyright
1497f17497SC.J. Collier *       notice, this list of conditions and the following disclaimer in
1597f17497SC.J. Collier *       the documentation and/or other materials provided with the
1697f17497SC.J. Collier *       distribution.
1797f17497SC.J. Collier *     * Neither the name of 6WIND S.A. nor the names of its
1897f17497SC.J. Collier *       contributors may be used to endorse or promote products derived
1997f17497SC.J. Collier *       from this software without specific prior written permission.
2097f17497SC.J. Collier *
3297f17497SC.J. Collier */
3397f17497SC.J. Collier
3497f17497SC.J. Collier#include <stddef.h>
3597f17497SC.J. Collier#include <assert.h>
3697f17497SC.J. Collier#include <stdint.h>
3797f17497SC.J. Collier#include <string.h>
3897f17497SC.J. Collier#include <errno.h>
3997f17497SC.J. Collier
4097f17497SC.J. Collier/* Verbs header. */
4197f17497SC.J. Collier/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
4297f17497SC.J. Collier#ifdef PEDANTIC
4332e04ea0SChristian Ehrhardt#pragma GCC diagnostic ignored "-Wpedantic"
4497f17497SC.J. Collier#endif
4597f17497SC.J. Collier#include <infiniband/verbs.h>
4697f17497SC.J. Collier#ifdef PEDANTIC
4732e04ea0SChristian Ehrhardt#pragma GCC diagnostic error "-Wpedantic"
4897f17497SC.J. Collier#endif
4997f17497SC.J. Collier
5097f17497SC.J. Collier/* DPDK headers don't like -pedantic. */
5197f17497SC.J. Collier#ifdef PEDANTIC
5232e04ea0SChristian Ehrhardt#pragma GCC diagnostic ignored "-Wpedantic"
5397f17497SC.J. Collier#endif
5497f17497SC.J. Collier#include <rte_ether.h>
5597f17497SC.J. Collier#include <rte_malloc.h>
5697f17497SC.J. Collier#include <rte_ethdev.h>
5797f17497SC.J. Collier#include <rte_common.h>
5897f17497SC.J. Collier#ifdef PEDANTIC
5932e04ea0SChristian Ehrhardt#pragma GCC diagnostic error "-Wpedantic"
6097f17497SC.J. Collier#endif
6197f17497SC.J. Collier
6297f17497SC.J. Collier#include "mlx5.h"
6397f17497SC.J. Collier#include "mlx5_rxtx.h"
6497f17497SC.J. Collier
6597f17497SC.J. Collierstruct fdir_flow_desc {
6697f17497SC.J. Collier	uint16_t dst_port;
6797f17497SC.J. Collier	uint16_t src_port;
6897f17497SC.J. Collier	uint32_t src_ip[4];
6997f17497SC.J. Collier	uint32_t dst_ip[4];
7097f17497SC.J. Collier	uint8_t	mac[6];
7197f17497SC.J. Collier	uint16_t vlan_tag;
7297f17497SC.J. Collier	enum hash_rxq_type type;
7397f17497SC.J. Collier};
7497f17497SC.J. Collier
7597f17497SC.J. Collierstruct mlx5_fdir_filter {
7697f17497SC.J. Collier	LIST_ENTRY(mlx5_fdir_filter) next;
7797f17497SC.J. Collier	uint16_t queue; /* Queue assigned to if FDIR match. */
7832e04ea0SChristian Ehrhardt	enum rte_eth_fdir_behavior behavior;
7997f17497SC.J. Collier	struct fdir_flow_desc desc;
8097f17497SC.J. Collier	struct ibv_exp_flow *flow;
8197f17497SC.J. Collier};
8297f17497SC.J. Collier
8397f17497SC.J. CollierLIST_HEAD(fdir_filter_list, mlx5_fdir_filter);
8497f17497SC.J. Collier
8597f17497SC.J. Collier/**
8697f17497SC.J. Collier * Convert struct rte_eth_fdir_filter to mlx5 filter descriptor.
8797f17497SC.J. Collier *
8897f17497SC.J. Collier * @param[in] fdir_filter
8997f17497SC.J. Collier *   DPDK filter structure to convert.
9097f17497SC.J. Collier * @param[out] desc
9197f17497SC.J. Collier *   Resulting mlx5 filter descriptor.
9297f17497SC.J. Collier * @param mode
9397f17497SC.J. Collier *   Flow director mode.
9497f17497SC.J. Collier */
9597f17497SC.J. Collierstatic void
9697f17497SC.J. Collierfdir_filter_to_flow_desc(const struct rte_eth_fdir_filter *fdir_filter,
9797f17497SC.J. Collier			 struct fdir_flow_desc *desc, enum rte_fdir_mode mode)
9897f17497SC.J. Collier{
9997f17497SC.J. Collier	/* Initialize descriptor. */
10097f17497SC.J. Collier	memset(desc, 0, sizeof(*desc));
10197f17497SC.J. Collier
10297f17497SC.J. Collier	/* Set VLAN ID. */
10397f17497SC.J. Collier	desc->vlan_tag = fdir_filter->input.flow_ext.vlan_tci;
10497f17497SC.J. Collier
10597f17497SC.J. Collier	/* Set MAC address. */
10697f17497SC.J. Collier	if (mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
10797f17497SC.J. Collier		rte_memcpy(desc->mac,
10897f17497SC.J. Collier			   fdir_filter->input.flow.mac_vlan_flow.mac_addr.
10997f17497SC.J. Collier				addr_bytes,
11097f17497SC.J. Collier			   sizeof(desc->mac));
11197f17497SC.J. Collier		desc->type = HASH_RXQ_ETH;
11297f17497SC.J. Collier		return;
11397f17497SC.J. Collier	}
11497f17497SC.J. Collier
11597f17497SC.J. Collier	/* Set mode */
11697f17497SC.J. Collier	switch (fdir_filter->input.flow_type) {
11797f17497SC.J. Collier	case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
11897f17497SC.J. Collier		desc->type = HASH_RXQ_UDPV4;
11997f17497SC.J. Collier		break;
12097f17497SC.J. Collier	case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
12197f17497SC.J. Collier		desc->type = HASH_RXQ_TCPV4;
12297f17497SC.J. Collier		break;
12397f17497SC.J. Collier	case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
12497f17497SC.J. Collier		desc->type = HASH_RXQ_IPV4;
12597f17497SC.J. Collier		break;
12697f17497SC.J. Collier	case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
12797f17497SC.J. Collier		desc->type = HASH_RXQ_UDPV6;
12897f17497SC.J. Collier		break;
12997f17497SC.J. Collier	case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
13097f17497SC.J. Collier		desc->type = HASH_RXQ_TCPV6;
13197f17497SC.J. Collier		break;
13297f17497SC.J. Collier	case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
13397f17497SC.J. Collier		desc->type = HASH_RXQ_IPV6;
13497f17497SC.J. Collier		break;
13597f17497SC.J. Collier	default:
13697f17497SC.J. Collier		break;
13797f17497SC.J. Collier	}
13897f17497SC.J. Collier
13997f17497SC.J. Collier	/* Set flow values */
14097f17497SC.J. Collier	switch (fdir_filter->input.flow_type) {
14197f17497SC.J. Collier	case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
14297f17497SC.J. Collier	case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
14397f17497SC.J. Collier		desc->src_port = fdir_filter->input.flow.udp4_flow.src_port;
14497f17497SC.J. Collier		desc->dst_port = fdir_filter->input.flow.udp4_flow.dst_port;
145fdd2322bSLuca Boccassi		/* fallthrough */
14697f17497SC.J. Collier	case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
14797f17497SC.J. Collier		desc->src_ip[0] = fdir_filter->input.flow.ip4_flow.src_ip;
14897f17497SC.J. Collier		desc->dst_ip[0] = fdir_filter->input.flow.ip4_flow.dst_ip;
14997f17497SC.J. Collier		break;
15097f17497SC.J. Collier	case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
15197f17497SC.J. Collier	case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
15297f17497SC.J. Collier		desc->src_port = fdir_filter->input.flow.udp6_flow.src_port;
15397f17497SC.J. Collier		desc->dst_port = fdir_filter->input.flow.udp6_flow.dst_port;
15497f17497SC.J. Collier		/* Fall through. */
15597f17497SC.J. Collier	case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
15697f17497SC.J. Collier		rte_memcpy(desc->src_ip,
15797f17497SC.J. Collier			   fdir_filter->input.flow.ipv6_flow.src_ip,
15897f17497SC.J. Collier			   sizeof(desc->src_ip));
15997f17497SC.J. Collier		rte_memcpy(desc->dst_ip,
16097f17497SC.J. Collier			   fdir_filter->input.flow.ipv6_flow.dst_ip,
16197f17497SC.J. Collier			   sizeof(desc->dst_ip));
16297f17497SC.J. Collier		break;
16397f17497SC.J. Collier	default:
16497f17497SC.J. Collier		break;
16597f17497SC.J. Collier	}
16697f17497SC.J. Collier}
16797f17497SC.J. Collier
16897f17497SC.J. Collier/**
16997f17497SC.J. Collier * Check if two flow descriptors overlap according to configured mask.
17097f17497SC.J. Collier *
17197f17497SC.J. Collier * @param priv
17297f17497SC.J. Collier *   Private structure that provides flow director mask.
17397f17497SC.J. Collier * @param desc1
17497f17497SC.J. Collier *   First flow descriptor to compare.
17597f17497SC.J. Collier * @param desc2
17697f17497SC.J. Collier *   Second flow descriptor to compare.
17797f17497SC.J. Collier *
17897f17497SC.J. Collier * @return
17997f17497SC.J. Collier *   Nonzero if descriptors overlap.
18097f17497SC.J. Collier */
18197f17497SC.J. Collierstatic int
18297f17497SC.J. Collierpriv_fdir_overlap(const struct priv *priv,
18397f17497SC.J. Collier		  const struct fdir_flow_desc *desc1,
18497f17497SC.J. Collier		  const struct fdir_flow_desc *desc2)
18597f17497SC.J. Collier{
18697f17497SC.J. Collier	const struct rte_eth_fdir_masks *mask =
18797f17497SC.J. Collier		&priv->dev->data->dev_conf.fdir_conf.mask;
18897f17497SC.J. Collier	unsigned int i;
18997f17497SC.J. Collier
19097f17497SC.J. Collier	if (desc1->type != desc2->type)
19197f17497SC.J. Collier		return 0;
19297f17497SC.J. Collier	/* Ignore non masked bits. */
19397f17497SC.J. Collier	for (i = 0; i != RTE_DIM(desc1->mac); ++i)
19497f17497SC.J. Collier		if ((desc1->mac[i] & mask->mac_addr_byte_mask) !=
19597f17497SC.J. Collier		    (desc2->mac[i] & mask->mac_addr_byte_mask))
19697f17497SC.J. Collier			return 0;
19797f17497SC.J. Collier	if (((desc1->src_port & mask->src_port_mask) !=
19897f17497SC.J. Collier	     (desc2->src_port & mask->src_port_mask)) ||
19997f17497SC.J. Collier	    ((desc1->dst_port & mask->dst_port_mask) !=
20097f17497SC.J. Collier	     (desc2->dst_port & mask->dst_port_mask)))
20197f17497SC.J. Collier		return 0;
20297f17497SC.J. Collier	switch (desc1->type) {
20397f17497SC.J. Collier	case HASH_RXQ_IPV4:
20497f17497SC.J. Collier	case HASH_RXQ_UDPV4:
20597f17497SC.J. Collier	case HASH_RXQ_TCPV4:
20697f17497SC.J. Collier		if (((desc1->src_ip[0] & mask->ipv4_mask.src_ip) !=
20797f17497SC.J. Collier		     (desc2->src_ip[0] & mask->ipv4_mask.src_ip)) ||
20897f17497SC.J. Collier		    ((desc1->dst_ip[0] & mask->ipv4_mask.dst_ip) !=
20997f17497SC.J. Collier		     (desc2->dst_ip[0] & mask->ipv4_mask.dst_ip)))
21097f17497SC.J. Collier			return 0;
21197f17497SC.J. Collier		break;
21297f17497SC.J. Collier	case HASH_RXQ_IPV6:
21397f17497SC.J. Collier	case HASH_RXQ_UDPV6:
21497f17497SC.J. Collier	case HASH_RXQ_TCPV6:
21597f17497SC.J. Collier		for (i = 0; i != RTE_DIM(desc1->src_ip); ++i)
21697f17497SC.J. Collier			if (((desc1->src_ip[i] & mask->ipv6_mask.src_ip[i]) !=
21797f17497SC.J. Collier			     (desc2->src_ip[i] & mask->ipv6_mask.src_ip[i])) ||
21897f17497SC.J. Collier			    ((desc1->dst_ip[i] & mask->ipv6_mask.dst_ip[i]) !=
21997f17497SC.J. Collier			     (desc2->dst_ip[i] & mask->ipv6_mask.dst_ip[i])))
22097f17497SC.J. Collier				return 0;
22197f17497SC.J. Collier		break;
22297f17497SC.J. Collier	default:
22397f17497SC.J. Collier		break;
22497f17497SC.J. Collier	}
22597f17497SC.J. Collier	return 1;
22697f17497SC.J. Collier}
22797f17497SC.J. Collier
22897f17497SC.J. Collier/**
22997f17497SC.J. Collier * Create flow director steering rule for a specific filter.
23097f17497SC.J. Collier *
23197f17497SC.J. Collier * @param priv
23297f17497SC.J. Collier *   Private structure.
23397f17497SC.J. Collier * @param mlx5_fdir_filter
23497f17497SC.J. Collier *   Filter to create a steering rule for.
23597f17497SC.J. Collier * @param fdir_queue
23697f17497SC.J. Collier *   Flow director queue for matching packets.
23797f17497SC.J. Collier *
23897f17497SC.J. Collier * @return
23997f17497SC.J. Collier *   0 on success, errno value on failure.
24097f17497SC.J. Collier */
24197f17497SC.J. Collierstatic int
24297f17497SC.J. Collierpriv_fdir_flow_add(struct priv *priv,
24397f17497SC.J. Collier		   struct mlx5_fdir_filter *mlx5_fdir_filter,
24497f17497SC.J. Collier		   struct fdir_queue *fdir_queue)
24597f17497SC.J. Collier{
24697f17497SC.J. Collier	struct ibv_exp_flow *flow;
24797f17497SC.J. Collier	struct fdir_flow_desc *desc = &mlx5_fdir_filter->desc;
24897f17497SC.J. Collier	enum rte_fdir_mode fdir_mode =
24997f17497SC.J. Collier		priv->dev->data->dev_conf.fdir_conf.mode;
25097f17497SC.J. Collier	struct rte_eth_fdir_masks *mask =
25197f17497SC.J. Collier		&priv->dev->data->dev_conf.fdir_conf.mask;
25297f17497SC.J. Collier	FLOW_ATTR_SPEC_ETH(data, priv_flow_attr(priv, NULL, 0, desc->type));
25397f17497SC.J. Collier	struct ibv_exp_flow_attr *attr = &data->attr;
25497f17497SC.J. Collier	uintptr_t spec_offset = (uintptr_t)&data->spec;
25597f17497SC.J. Collier	struct ibv_exp_flow_spec_eth *spec_eth;
25697f17497SC.J. Collier	struct ibv_exp_flow_spec_ipv4 *spec_ipv4;
25797f17497SC.J. Collier	struct ibv_exp_flow_spec_ipv6 *spec_ipv6;
25897f17497SC.J. Collier	struct ibv_exp_flow_spec_tcp_udp *spec_tcp_udp;
25997f17497SC.J. Collier	struct mlx5_fdir_filter *iter_fdir_filter;
26097f17497SC.J. Collier	unsigned int i;
26197f17497SC.J. Collier
26297f17497SC.J. Collier	/* Abort if an existing flow overlaps this one to avoid packet
26397f17497SC.J. Collier	 * duplication, even if it targets another queue. */
26497f17497SC.J. Collier	LIST_FOREACH(iter_fdir_filter, priv->fdir_filter_list, next)
26597f17497SC.J. Collier		if ((iter_fdir_filter != mlx5_fdir_filter) &&
26697f17497SC.J. Collier		    (iter_fdir_filter->flow != NULL) &&
26797f17497SC.J. Collier		    (priv_fdir_overlap(priv,
26897f17497SC.J. Collier				       &mlx5_fdir_filter->desc,
26997f17497SC.J. Collier				       &iter_fdir_filter->desc)))
27097f17497SC.J. Collier			return EEXIST;
27197f17497SC.J. Collier
27297f17497SC.J. Collier	/*
27397f17497SC.J. Collier	 * No padding must be inserted by the compiler between attr and spec.
27497f17497SC.J. Collier	 * This layout is expected by libibverbs.
27597f17497SC.J. Collier	 */
27697f17497SC.J. Collier	assert(((uint8_t *)attr + sizeof(*attr)) == (uint8_t *)spec_offset);
27797f17497SC.J. Collier	priv_flow_attr(priv, attr, sizeof(data), desc->type);
27897f17497SC.J. Collier
27997f17497SC.J. Collier	/* Set Ethernet spec */
28097f17497SC.J. Collier	spec_eth = (struct ibv_exp_flow_spec_eth *)spec_offset;
28197f17497SC.J. Collier
28297f17497SC.J. Collier	/* The first specification must be Ethernet. */
28397f17497SC.J. Collier	assert(spec_eth->type == IBV_EXP_FLOW_SPEC_ETH);
28497f17497SC.J. Collier	assert(spec_eth->size == sizeof(*spec_eth));
28597f17497SC.J. Collier
28697f17497SC.J. Collier	/* VLAN ID */
28797f17497SC.J. Collier	spec_eth->val.vlan_tag = desc->vlan_tag & mask->vlan_tci_mask;
28897f17497SC.J. Collier	spec_eth->mask.vlan_tag = mask->vlan_tci_mask;
28997f17497SC.J. Collier
29097f17497SC.J. Collier	/* Update priority */
29197f17497SC.J. Collier	attr->priority = 2;
29297f17497SC.J. Collier
29397f17497SC.J. Collier	if (fdir_mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
29497f17497SC.J. Collier		/* MAC Address */
29597f17497SC.J. Collier		for (i = 0; i != RTE_DIM(spec_eth->mask.dst_mac); ++i) {
29697f17497SC.J. Collier			spec_eth->val.dst_mac[i] =
29797f17497SC.J. Collier				desc->mac[i] & mask->mac_addr_byte_mask;
29897f17497SC.J. Collier			spec_eth->mask.dst_mac[i] = mask->mac_addr_byte_mask;
29997f17497SC.J. Collier		}
30097f17497SC.J. Collier		goto create_flow;
30197f17497SC.J. Collier	}
30297f17497SC.J. Collier
30397f17497SC.J. Collier	switch (desc->type) {
30497f17497SC.J. Collier	case HASH_RXQ_IPV4:
30597f17497SC.J. Collier	case HASH_RXQ_UDPV4:
30697f17497SC.J. Collier	case HASH_RXQ_TCPV4:
30797f17497SC.J. Collier		spec_offset += spec_eth->size;
30897f17497SC.J. Collier
30997f17497SC.J. Collier		/* Set IP spec */
31097f17497SC.J. Collier		spec_ipv4 = (struct ibv_exp_flow_spec_ipv4 *)spec_offset;
31197f17497SC.J. Collier
31297f17497SC.J. Collier		/* The second specification must be IP. */
31397f17497SC.J. Collier		assert(spec_ipv4->type == IBV_EXP_FLOW_SPEC_IPV4);
31497f17497SC.J. Collier		assert(spec_ipv4->size == sizeof(*spec_ipv4));
31597f17497SC.J. Collier
31697f17497SC.J. Collier		spec_ipv4->val.src_ip =
31797f17497SC.J. Collier			desc->src_ip[0] & mask->ipv4_mask.src_ip;
31897f17497SC.J. Collier		spec_ipv4->val.dst_ip =
31997f17497SC.J. Collier			desc->dst_ip[0] & mask->ipv4_mask.dst_ip;
32097f17497SC.J. Collier		spec_ipv4->mask.src_ip = mask->ipv4_mask.src_ip;
32197f17497SC.J. Collier		spec_ipv4->mask.dst_ip = mask->ipv4_mask.dst_ip;
32297f17497SC.J. Collier
32397f17497SC.J. Collier		/* Update priority */
32497f17497SC.J. Collier		attr->priority = 1;
32597f17497SC.J. Collier
32697f17497SC.J. Collier		if (desc->type == HASH_RXQ_IPV4)
32797f17497SC.J. Collier			goto create_flow;
32897f17497SC.J. Collier
32997f17497SC.J. Collier		spec_offset += spec_ipv4->size;
33097f17497SC.J. Collier		break;
33197f17497SC.J. Collier	case HASH_RXQ_IPV6:
33297f17497SC.J. Collier	case HASH_RXQ_UDPV6:
33397f17497SC.J. Collier	case HASH_RXQ_TCPV6:
33497f17497SC.J. Collier		spec_offset += spec_eth->size;
33597f17497SC.J. Collier
33697f17497SC.J. Collier		/* Set IP spec */
33797f17497SC.J. Collier		spec_ipv6 = (struct ibv_exp_flow_spec_ipv6 *)spec_offset;
33897f17497SC.J. Collier
33997f17497SC.J. Collier		/* The second specification must be IP. */
34097f17497SC.J. Collier		assert(spec_ipv6->type == IBV_EXP_FLOW_SPEC_IPV6);
34197f17497SC.J. Collier		assert(spec_ipv6->size == sizeof(*spec_ipv6));
34297f17497SC.J. Collier
34397f17497SC.J. Collier		for (i = 0; i != RTE_DIM(desc->src_ip); ++i) {
34497f17497SC.J. Collier			((uint32_t *)spec_ipv6->val.src_ip)[i] =
34597f17497SC.J. Collier				desc->src_ip[i] & mask->ipv6_mask.src_ip[i];
34697f17497SC.J. Collier			((uint32_t *)spec_ipv6->val.dst_ip)[i] =
34797f17497SC.J. Collier				desc->dst_ip[i] & mask->ipv6_mask.dst_ip[i];
34897f17497SC.J. Collier		}
34997f17497SC.J. Collier		rte_memcpy(spec_ipv6->mask.src_ip,
35097f17497SC.J. Collier			   mask->ipv6_mask.src_ip,
35197f17497SC.J. Collier			   sizeof(spec_ipv6->mask.src_ip));
35297f17497SC.J. Collier		rte_memcpy(spec_ipv6->mask.dst_ip,
35397f17497SC.J. Collier			   mask->ipv6_mask.dst_ip,
35497f17497SC.J. Collier			   sizeof(spec_ipv6->mask.dst_ip));
35597f17497SC.J. Collier
35697f17497SC.J. Collier		/* Update priority */
35797f17497SC.J. Collier		attr->priority = 1;
35897f17497SC.J. Collier
35997f17497SC.J. Collier		if (desc->type == HASH_RXQ_IPV6)
36097f17497SC.J. Collier			goto create_flow;
36197f17497SC.J. Collier
36297f17497SC.J. Collier		spec_offset += spec_ipv6->size;
36397f17497SC.J. Collier		break;
36497f17497SC.J. Collier	default:
36597f17497SC.J. Collier		ERROR("invalid flow attribute type");
36697f17497SC.J. Collier		return EINVAL;
36797f17497SC.J. Collier	}
36897f17497SC.J. Collier
36997f17497SC.J. Collier	/* Set TCP/UDP flow specification. */
37097f17497SC.J. Collier	spec_tcp_udp = (struct ibv_exp_flow_spec_tcp_udp *)spec_offset;
37197f17497SC.J. Collier
37297f17497SC.J. Collier	/* The third specification must be TCP/UDP. */
37397f17497SC.J. Collier	assert(spec_tcp_udp->type == IBV_EXP_FLOW_SPEC_TCP ||
37497f17497SC.J. Collier	       spec_tcp_udp->type == IBV_EXP_FLOW_SPEC_UDP);
37597f17497SC.J. Collier	assert(spec_tcp_udp->size == sizeof(*spec_tcp_udp));
37697f17497SC.J. Collier
37797f17497SC.J. Collier	spec_tcp_udp->val.src_port = desc->src_port & mask->src_port_mask;
37897f17497SC.J. Collier	spec_tcp_udp->val.dst_port = desc->dst_port & mask->dst_port_mask;
37997f17497SC.J. Collier	spec_tcp_udp->mask.src_port = mask->src_port_mask;
38097f17497SC.J. Collier	spec_tcp_udp->mask.dst_port = mask->dst_port_mask;
38197f17497SC.J. Collier
38297f17497SC.J. Collier	/* Update priority */
38397f17497SC.J. Collier	attr->priority = 0;
38497f17497SC.J. Collier
38597f17497SC.J. Colliercreate_flow:
38697f17497SC.J. Collier
38797f17497SC.J. Collier	errno = 0;
38897f17497SC.J. Collier	flow = ibv_exp_create_flow(fdir_queue->qp, attr);
38997f17497SC.J. Collier	if (flow == NULL) {
39097f17497SC.J. Collier		/* It's not clear whether errno is always set in this case. */
39197f17497SC.J. Collier		ERROR("%p: flow director configuration failed, errno=%d: %s",
39297f17497SC.J. Collier		      (void *)priv, errno,
39397f17497SC.J. Collier		      (errno ? strerror(errno) : "Unknown error"));
39497f17497SC.J. Collier		if (errno)
39597f17497SC.J. Collier			return errno;
39697f17497SC.J. Collier		return EINVAL;
39797f17497SC.J. Collier	}
39897f17497SC.J. Collier
39997f17497SC.J. Collier	DEBUG("%p: added flow director rule (%p)", (void *)priv, (void *)flow);
40097f17497SC.J. Collier	mlx5_fdir_filter->flow = flow;
40197f17497SC.J. Collier	return 0;
40297f17497SC.J. Collier}
40397f17497SC.J. Collier
40432e04ea0SChristian Ehrhardt/**
40532e04ea0SChristian Ehrhardt * Destroy a flow director queue.
40632e04ea0SChristian Ehrhardt *
40732e04ea0SChristian Ehrhardt * @param fdir_queue
40832e04ea0SChristian Ehrhardt *   Flow director queue to be destroyed.
40932e04ea0SChristian Ehrhardt */
41032e04ea0SChristian Ehrhardtvoid
41132e04ea0SChristian Ehrhardtpriv_fdir_queue_destroy(struct priv *priv, struct fdir_queue *fdir_queue)
41232e04ea0SChristian Ehrhardt{
41332e04ea0SChristian Ehrhardt	struct mlx5_fdir_filter *fdir_filter;
41432e04ea0SChristian Ehrhardt
41532e04ea0SChristian Ehrhardt	/* Disable filter flows still applying to this queue. */
41632e04ea0SChristian Ehrhardt	LIST_FOREACH(fdir_filter, priv->fdir_filter_list, next) {
41732e04ea0SChristian Ehrhardt		unsigned int idx = fdir_filter->queue;
41832e04ea0SChristian Ehrhardt		struct rxq_ctrl *rxq_ctrl =
41932e04ea0SChristian Ehrhardt			container_of((*priv->rxqs)[idx], struct rxq_ctrl, rxq);
42032e04ea0SChristian Ehrhardt
42132e04ea0SChristian Ehrhardt		assert(idx < priv->rxqs_n);
42232e04ea0SChristian Ehrhardt		if (fdir_queue == rxq_ctrl->fdir_queue &&
42332e04ea0SChristian Ehrhardt		    fdir_filter->flow != NULL) {
42432e04ea0SChristian Ehrhardt			claim_zero(ibv_exp_destroy_flow(fdir_filter->flow));
42532e04ea0SChristian Ehrhardt			fdir_filter->flow = NULL;
42632e04ea0SChristian Ehrhardt		}
42732e04ea0SChristian Ehrhardt	}
42832e04ea0SChristian Ehrhardt	assert(fdir_queue->qp);
42932e04ea0SChristian Ehrhardt	claim_zero(ibv_destroy_qp(fdir_queue->qp));
43032e04ea0SChristian Ehrhardt	assert(fdir_queue->ind_table);
43132e04ea0SChristian Ehrhardt	claim_zero(ibv_exp_destroy_rwq_ind_table(fdir_queue->ind_table));
43232e04ea0SChristian Ehrhardt	if (fdir_queue->wq)
43332e04ea0SChristian Ehrhardt		claim_zero(ibv_exp_destroy_wq(fdir_queue->wq));
43432e04ea0SChristian Ehrhardt	if (fdir_queue->cq)
43532e04ea0SChristian Ehrhardt		claim_zero(ibv_destroy_cq(fdir_queue->cq));
43632e04ea0SChristian Ehrhardt#ifndef NDEBUG
43732e04ea0SChristian Ehrhardt	memset(fdir_queue, 0x2a, sizeof(*fdir_queue));
43832e04ea0SChristian Ehrhardt#endif
43932e04ea0SChristian Ehrhardt	rte_free(fdir_queue);
44032e04ea0SChristian Ehrhardt}
44132e04ea0SChristian Ehrhardt
44232e04ea0SChristian Ehrhardt/**
44332e04ea0SChristian Ehrhardt * Create a flow director queue.
44432e04ea0SChristian Ehrhardt *
44532e04ea0SChristian Ehrhardt * @param priv
44632e04ea0SChristian Ehrhardt *   Private structure.
44732e04ea0SChristian Ehrhardt * @param wq
44832e04ea0SChristian Ehrhardt *   Work queue to route matched packets to, NULL if one needs to
44932e04ea0SChristian Ehrhardt *   be created.
45032e04ea0SChristian Ehrhardt *
45132e04ea0SChristian Ehrhardt * @return
45232e04ea0SChristian Ehrhardt *   Related flow director queue on success, NULL otherwise.
45332e04ea0SChristian Ehrhardt */
45432e04ea0SChristian Ehrhardtstatic struct fdir_queue *
45532e04ea0SChristian Ehrhardtpriv_fdir_queue_create(struct priv *priv, struct ibv_exp_wq *wq,
45632e04ea0SChristian Ehrhardt		       unsigned int socket)
45732e04ea0SChristian Ehrhardt{
45832e04ea0SChristian Ehrhardt	struct fdir_queue *fdir_queue;
45932e04ea0SChristian Ehrhardt
46032e04ea0SChristian Ehrhardt	fdir_queue = rte_calloc_socket(__func__, 1, sizeof(*fdir_queue),
46132e04ea0SChristian Ehrhardt				       0, socket);
46232e04ea0SChristian Ehrhardt	if (!fdir_queue) {
46332e04ea0SChristian Ehrhardt		ERROR("cannot allocate flow director queue");
46432e04ea0SChristian Ehrhardt		return NULL;
46532e04ea0SChristian Ehrhardt	}
46632e04ea0SChristian Ehrhardt	assert(priv->pd);
46732e04ea0SChristian Ehrhardt	assert(priv->ctx);
46832e04ea0SChristian Ehrhardt	if (!wq) {
46932e04ea0SChristian Ehrhardt		fdir_queue->cq = ibv_exp_create_cq(
47032e04ea0SChristian Ehrhardt			priv->ctx, 1, NULL, NULL, 0,
47132e04ea0SChristian Ehrhardt			&(struct ibv_exp_cq_init_attr){
47232e04ea0SChristian Ehrhardt				.comp_mask = 0,
47332e04ea0SChristian Ehrhardt			});
47432e04ea0SChristian Ehrhardt		if (!fdir_queue->cq) {
47532e04ea0SChristian Ehrhardt			ERROR("cannot create flow director CQ");
47632e04ea0SChristian Ehrhardt			goto error;
47732e04ea0SChristian Ehrhardt		}
47832e04ea0SChristian Ehrhardt		fdir_queue->wq = ibv_exp_create_wq(
47932e04ea0SChristian Ehrhardt			priv->ctx,
48032e04ea0SChristian Ehrhardt			&(struct ibv_exp_wq_init_attr){
48132e04ea0SChristian Ehrhardt				.wq_type = IBV_EXP_WQT_RQ,
48232e04ea0SChristian Ehrhardt				.max_recv_wr = 1,
48332e04ea0SChristian Ehrhardt				.max_recv_sge = 1,
48432e04ea0SChristian Ehrhardt				.pd = priv->pd,
48532e04ea0SChristian Ehrhardt				.cq = fdir_queue->cq,
48632e04ea0SChristian Ehrhardt			});
48732e04ea0SChristian Ehrhardt		if (!fdir_queue->wq) {
48832e04ea0SChristian Ehrhardt			ERROR("cannot create flow director WQ");
48932e04ea0SChristian Ehrhardt			goto error;
49032e04ea0SChristian Ehrhardt		}
49132e04ea0SChristian Ehrhardt		wq = fdir_queue->wq;
49232e04ea0SChristian Ehrhardt	}
49332e04ea0SChristian Ehrhardt	fdir_queue->ind_table = ibv_exp_create_rwq_ind_table(
49432e04ea0SChristian Ehrhardt		priv->ctx,
49532e04ea0SChristian Ehrhardt		&(struct ibv_exp_rwq_ind_table_init_attr){
49632e04ea0SChristian Ehrhardt			.pd = priv->pd,
49732e04ea0SChristian Ehrhardt			.log_ind_tbl_size = 0,
49832e04ea0SChristian Ehrhardt			.ind_tbl = &wq,
49932e04ea0SChristian Ehrhardt			.comp_mask = 0,
50032e04ea0SChristian Ehrhardt		});
50132e04ea0SChristian Ehrhardt	if (!fdir_queue->ind_table) {
50232e04ea0SChristian Ehrhardt		ERROR("cannot create flow director indirection table");
50332e04ea0SChristian Ehrhardt		goto error;
50432e04ea0SChristian Ehrhardt	}
50532e04ea0SChristian Ehrhardt	fdir_queue->qp = ibv_exp_create_qp(
50632e04ea0SChristian Ehrhardt		priv->ctx,
50732e04ea0SChristian Ehrhardt		&(struct ibv_exp_qp_init_attr){
50832e04ea0SChristian Ehrhardt			.qp_type = IBV_QPT_RAW_PACKET,
50932e04ea0SChristian Ehrhardt			.comp_mask =
51032e04ea0SChristian Ehrhardt				IBV_EXP_QP_INIT_ATTR_PD |
51132e04ea0SChristian Ehrhardt				IBV_EXP_QP_INIT_ATTR_PORT |
51232e04ea0SChristian Ehrhardt				IBV_EXP_QP_INIT_ATTR_RX_HASH,
51332e04ea0SChristian Ehrhardt			.pd = priv->pd,
51432e04ea0SChristian Ehrhardt			.rx_hash_conf = &(struct ibv_exp_rx_hash_conf){
51532e04ea0SChristian Ehrhardt				.rx_hash_function =
51632e04ea0SChristian Ehrhardt					IBV_EXP_RX_HASH_FUNC_TOEPLITZ,
51732e04ea0SChristian Ehrhardt				.rx_hash_key_len = rss_hash_default_key_len,
51832e04ea0SChristian Ehrhardt				.rx_hash_key = rss_hash_default_key,
51932e04ea0SChristian Ehrhardt				.rx_hash_fields_mask = 0,
52032e04ea0SChristian Ehrhardt				.rwq_ind_tbl = fdir_queue->ind_table,
52132e04ea0SChristian Ehrhardt			},
52232e04ea0SChristian Ehrhardt			.port_num = priv->port,
52332e04ea0SChristian Ehrhardt		});
52432e04ea0SChristian Ehrhardt	if (!fdir_queue->qp) {
52532e04ea0SChristian Ehrhardt		ERROR("cannot create flow director hash RX QP");
52632e04ea0SChristian Ehrhardt		goto error;
52732e04ea0SChristian Ehrhardt	}
52832e04ea0SChristian Ehrhardt	return fdir_queue;
52932e04ea0SChristian Ehrhardterror:
53032e04ea0SChristian Ehrhardt	assert(fdir_queue);
53132e04ea0SChristian Ehrhardt	assert(!fdir_queue->qp);
53232e04ea0SChristian Ehrhardt	if (fdir_queue->ind_table)
53332e04ea0SChristian Ehrhardt		claim_zero(ibv_exp_destroy_rwq_ind_table
53432e04ea0SChristian Ehrhardt			   (fdir_queue->ind_table));
53532e04ea0SChristian Ehrhardt	if (fdir_queue->wq)
53632e04ea0SChristian Ehrhardt		claim_zero(ibv_exp_destroy_wq(fdir_queue->wq));
53732e04ea0SChristian Ehrhardt	if (fdir_queue->cq)
53832e04ea0SChristian Ehrhardt		claim_zero(ibv_destroy_cq(fdir_queue->cq));
53932e04ea0SChristian Ehrhardt	rte_free(fdir_queue);
54032e04ea0SChristian Ehrhardt	return NULL;
54132e04ea0SChristian Ehrhardt}
54232e04ea0SChristian Ehrhardt
54397f17497SC.J. Collier/**
54497f17497SC.J. Collier * Get flow director queue for a specific RX queue, create it in case
54597f17497SC.J. Collier * it does not exist.
54697f17497SC.J. Collier *
54797f17497SC.J. Collier * @param priv
54897f17497SC.J. Collier *   Private structure.
54997f17497SC.J. Collier * @param idx
55097f17497SC.J. Collier *   RX queue index.
55197f17497SC.J. Collier *
55297f17497SC.J. Collier * @return
55397f17497SC.J. Collier *   Related flow director queue on success, NULL otherwise.
55497f17497SC.J. Collier */
55597f17497SC.J. Collierstatic struct fdir_queue *
55697f17497SC.J. Collierpriv_get_fdir_queue(struct priv *priv, uint16_t idx)
55797f17497SC.J. Collier{
5588b25d1adSChristian Ehrhardt	struct rxq_ctrl *rxq_ctrl =
5598b25d1adSChristian Ehrhardt		container_of((*priv->rxqs)[idx], struct rxq_ctrl, rxq);
56032e04ea0SChristian Ehrhardt	struct fdir_queue *fdir_queue = rxq_ctrl->fdir_queue;
56197f17497SC.J. Collier
56232e04ea0SChristian Ehrhardt	assert(rxq_ctrl->wq);
56332e04ea0SChristian Ehrhardt	if (fdir_queue == NULL) {
56432e04ea0SChristian Ehrhardt		fdir_queue = priv_fdir_queue_create(priv, rxq_ctrl->wq,
56532e04ea0SChristian Ehrhardt						    rxq_ctrl->socket);
56632e04ea0SChristian Ehrhardt		rxq_ctrl->fdir_queue = fdir_queue;
56797f17497SC.J. Collier	}
56897f17497SC.J. Collier	return fdir_queue;
56932e04ea0SChristian Ehrhardt}
57097f17497SC.J. Collier
57132e04ea0SChristian Ehrhardt/**
57232e04ea0SChristian Ehrhardt * Get or flow director drop queue. Create it if it does not exist.
57332e04ea0SChristian Ehrhardt *
57432e04ea0SChristian Ehrhardt * @param priv
57532e04ea0SChristian Ehrhardt *   Private structure.
57632e04ea0SChristian Ehrhardt *
57732e04ea0SChristian Ehrhardt * @return
57832e04ea0SChristian Ehrhardt *   Flow director drop queue on success, NULL otherwise.
57932e04ea0SChristian Ehrhardt */
58032e04ea0SChristian Ehrhardtstatic struct fdir_queue *
58132e04ea0SChristian Ehrhardtpriv_get_fdir_drop_queue(struct priv *priv)
58232e04ea0SChristian Ehrhardt{
58332e04ea0SChristian Ehrhardt	struct fdir_queue *fdir_queue = priv->fdir_drop_queue;
58497f17497SC.J. Collier
58532e04ea0SChristian Ehrhardt	if (fdir_queue == NULL) {
58632e04ea0SChristian Ehrhardt		unsigned int socket = SOCKET_ID_ANY;
58732e04ea0SChristian Ehrhardt
58832e04ea0SChristian Ehrhardt		/* Select a known NUMA socket if possible. */
58932e04ea0SChristian Ehrhardt		if (priv->rxqs_n && (*priv->rxqs)[0])
59032e04ea0SChristian Ehrhardt			socket = container_of((*priv->rxqs)[0],
59132e04ea0SChristian Ehrhardt					      struct rxq_ctrl, rxq)->socket;
59232e04ea0SChristian Ehrhardt		fdir_queue = priv_fdir_queue_create(priv, NULL, socket);
59332e04ea0SChristian Ehrhardt		priv->fdir_drop_queue = fdir_queue;
59432e04ea0SChristian Ehrhardt	}
59532e04ea0SChristian Ehrhardt	return fdir_queue;
59697f17497SC.J. Collier}
59797f17497SC.J. Collier
59897f17497SC.J. Collier/**
59997f17497SC.J. Collier * Enable flow director filter and create steering rules.
60097f17497SC.J. Collier *
60197f17497SC.J. Collier * @param priv
60297f17497SC.J. Collier *   Private structure.
60397f17497SC.J. Collier * @param mlx5_fdir_filter
60497f17497SC.J. Collier *   Filter to create steering rule for.
60597f17497SC.J. Collier *
60697f17497SC.J. Collier * @return
60797f17497SC.J. Collier *   0 on success, errno value on failure.
60897f17497SC.J. Collier */
60997f17497SC.J. Collierstatic int
61097f17497SC.J. Collierpriv_fdir_filter_enable(struct priv *priv,
61197f17497SC.J. Collier			struct mlx5_fdir_filter *mlx5_fdir_filter)
61297f17497SC.J. Collier{
61397f17497SC.J. Collier	struct fdir_queue *fdir_queue;
61497f17497SC.J. Collier
61597f17497SC.J. Collier	/* Check if flow already exists. */
61697f17497SC.J. Collier	if (mlx5_fdir_filter->flow != NULL)
61797f17497SC.J. Collier		return 0;
61897f17497SC.J. Collier
61997f17497SC.J. Collier	/* Get fdir_queue for specific queue. */
62032e04ea0SChristian Ehrhardt	if (mlx5_fdir_filter->behavior == RTE_ETH_FDIR_REJECT)
62132e04ea0SChristian Ehrhardt		fdir_queue = priv_get_fdir_drop_queue(priv);
62232e04ea0SChristian Ehrhardt	else
62332e04ea0SChristian Ehrhardt		fdir_queue = priv_get_fdir_queue(priv,
62432e04ea0SChristian Ehrhardt						 mlx5_fdir_filter->queue);
62597f17497SC.J. Collier
62697f17497SC.J. Collier	if (fdir_queue == NULL) {
62797f17497SC.J. Collier		ERROR("failed to create flow director rxq for queue %d",
62897f17497SC.J. Collier		      mlx5_fdir_filter->queue);
62997f17497SC.J. Collier		return EINVAL;
63097f17497SC.J. Collier	}
63197f17497SC.J. Collier
63297f17497SC.J. Collier	/* Create flow */
63397f17497SC.J. Collier	return priv_fdir_flow_add(priv, mlx5_fdir_filter, fdir_queue);
63497f17497SC.J. Collier}
63597f17497SC.J. Collier
63697f17497SC.J. Collier/**
63797f17497SC.J. Collier * Initialize flow director filters list.
63897f17497SC.J. Collier *
63997f17497SC.J. Collier * @param priv
64097f17497SC.J. Collier *   Private structure.
64197f17497SC.J. Collier *
64297f17497SC.J. Collier * @return
64397f17497SC.J. Collier *   0 on success, errno value on failure.
64497f17497SC.J. Collier */
64597f17497SC.J. Collierint
64697f17497SC.J. Collierfdir_init_filters_list(struct priv *priv)
64797f17497SC.J. Collier{
64897f17497SC.J. Collier	/* Filter list initialization should be done only once. */
64997f17497SC.J. Collier	if (priv->fdir_filter_list)
65097f17497SC.J. Collier		return 0;
65197f17497SC.J. Collier
65297f17497SC.J. Collier	/* Create filters list. */
65397f17497SC.J. Collier	priv->fdir_filter_list =
65497f17497SC.J. Collier		rte_calloc(__func__, 1, sizeof(*priv->fdir_filter_list), 0);
65597f17497SC.J. Collier
65697f17497SC.J. Collier	if (priv->fdir_filter_list == NULL) {
65797f17497SC.J. Collier		int err = ENOMEM;
65897f17497SC.J. Collier
65997f17497SC.J. Collier		ERROR("cannot allocate flow director filter list: %s",
66097f17497SC.J. Collier		      strerror(err));
66197f17497SC.J. Collier		return err;
66297f17497SC.J. Collier	}
66397f17497SC.J. Collier
66497f17497SC.J. Collier	LIST_INIT(priv->fdir_filter_list);
66597f17497SC.J. Collier
66697f17497SC.J. Collier	return 0;
66797f17497SC.J. Collier}
66897f17497SC.J. Collier
66997f17497SC.J. Collier/**
67097f17497SC.J. Collier * Flush all filters.
67197f17497SC.J. Collier *
67297f17497SC.J. Collier * @param priv
67397f17497SC.J. Collier *   Private structure.
67497f17497SC.J. Collier */
67597f17497SC.J. Collierstatic void
67697f17497SC.J. Collierpriv_fdir_filter_flush(struct priv *priv)
67797f17497SC.J. Collier{
67897f17497SC.J. Collier	struct mlx5_fdir_filter *mlx5_fdir_filter;
67997f17497SC.J. Collier
68097f17497SC.J. Collier	while ((mlx5_fdir_filter = LIST_FIRST(priv->fdir_filter_list))) {
68197f17497SC.J. Collier		struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
68297f17497SC.J. Collier
68397f17497SC.J. Collier		DEBUG("%p: flushing flow director filter %p",
68497f17497SC.J. Collier		      (void *)priv, (void *)mlx5_fdir_filter);
68597f17497SC.J. Collier		LIST_REMOVE(mlx5_fdir_filter, next);
68697f17497SC.J. Collier		if (flow != NULL)
68797f17497SC.J. Collier			claim_zero(ibv_exp_destroy_flow(flow));
68897f17497SC.J. Collier		rte_free(mlx5_fdir_filter);
68997f17497SC.J. Collier	}
69097f17497SC.J. Collier}
69197f17497SC.J. Collier
69297f17497SC.J. Collier/**
69397f17497SC.J. Collier * Remove all flow director filters and delete list.
69497f17497SC.J. Collier *
69597f17497SC.J. Collier * @param priv
69697f17497SC.J. Collier *   Private structure.
69797f17497SC.J. Collier */
69897f17497SC.J. Colliervoid
69997f17497SC.J. Collierpriv_fdir_delete_filters_list(struct priv *priv)
70097f17497SC.J. Collier{
70197f17497SC.J. Collier	priv_fdir_filter_flush(priv);
70297f17497SC.J. Collier	rte_free(priv->fdir_filter_list);
70397f17497SC.J. Collier	priv->fdir_filter_list = NULL;
70497f17497SC.J. Collier}
70597f17497SC.J. Collier
70697f17497SC.J. Collier/**
70797f17497SC.J. Collier * Disable flow director, remove all steering rules.
70897f17497SC.J. Collier *
70997f17497SC.J. Collier * @param priv
71097f17497SC.J. Collier *   Private structure.
71197f17497SC.J. Collier */
71297f17497SC.J. Colliervoid
71397f17497SC.J. Collierpriv_fdir_disable(struct priv *priv)
71497f17497SC.J. Collier{
71597f17497SC.J. Collier	unsigned int i;
71697f17497SC.J. Collier	struct mlx5_fdir_filter *mlx5_fdir_filter;
71797f17497SC.J. Collier
71897f17497SC.J. Collier	/* Run on every flow director filter and destroy flow handle. */
71997f17497SC.J. Collier	LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
72097f17497SC.J. Collier		struct ibv_exp_flow *flow;
72197f17497SC.J. Collier
72297f17497SC.J. Collier		/* Only valid elements should be in the list */
72397f17497SC.J. Collier		assert(mlx5_fdir_filter != NULL);
72497f17497SC.J. Collier		flow = mlx5_fdir_filter->flow;
72597f17497SC.J. Collier
72697f17497SC.J. Collier		/* Destroy flow handle */
72797f17497SC.J. Collier		if (flow != NULL) {
72897f17497SC.J. Collier			claim_zero(ibv_exp_destroy_flow(flow));
72997f17497SC.J. Collier			mlx5_fdir_filter->flow = NULL;
73097f17497SC.J. Collier		}
73197f17497SC.J. Collier	}
73297f17497SC.J. Collier
73332e04ea0SChristian Ehrhardt	/* Destroy flow director context in each RX queue. */
73497f17497SC.J. Collier	for (i = 0; (i != priv->rxqs_n); i++) {
735fdd2322bSLuca Boccassi		struct rxq_ctrl *rxq_ctrl;
73697f17497SC.J. Collier
737fdd2322bSLuca Boccassi		if (!(*priv->rxqs)[i])
738fdd2322bSLuca Boccassi			continue;
739fdd2322bSLuca Boccassi		rxq_ctrl = container_of((*priv->rxqs)[i], struct rxq_ctrl, rxq);
74032e04ea0SChristian Ehrhardt		if (!rxq_ctrl->fdir_queue)
74132e04ea0SChristian Ehrhardt			continue;
74232e04ea0SChristian Ehrhardt		priv_fdir_queue_destroy(priv, rxq_ctrl->fdir_queue);
74332e04ea0SChristian Ehrhardt		rxq_ctrl->fdir_queue = NULL;
74432e04ea0SChristian Ehrhardt	}
74532e04ea0SChristian Ehrhardt	if (priv->fdir_drop_queue) {
74632e04ea0SChristian Ehrhardt		priv_fdir_queue_destroy(priv, priv->fdir_drop_queue);
74732e04ea0SChristian Ehrhardt		priv->fdir_drop_queue = NULL;
74897f17497SC.J. Collier	}
74997f17497SC.J. Collier}
75097f17497SC.J. Collier
75197f17497SC.J. Collier/**
75297f17497SC.J. Collier * Enable flow director, create steering rules.
75397f17497SC.J. Collier *
75497f17497SC.J. Collier * @param priv
75597f17497SC.J. Collier *   Private structure.
75697f17497SC.J. Collier */
75797f17497SC.J. Colliervoid
75897f17497SC.J. Collierpriv_fdir_enable(struct priv *priv)
75997f17497SC.J. Collier{
76097f17497SC.J. Collier	struct mlx5_fdir_filter *mlx5_fdir_filter;
76197f17497SC.J. Collier
76297f17497SC.J. Collier	/* Run on every fdir filter and create flow handle */
76397f17497SC.J. Collier	LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
76497f17497SC.J. Collier		/* Only valid elements should be in the list */
76597f17497SC.J. Collier		assert(mlx5_fdir_filter != NULL);
76697f17497SC.J. Collier
76797f17497SC.J. Collier		priv_fdir_filter_enable(priv, mlx5_fdir_filter);
76897f17497SC.J. Collier	}
76997f17497SC.J. Collier}
77097f17497SC.J. Collier
77197f17497SC.J. Collier/**
77297f17497SC.J. Collier * Find specific filter in list.
77397f17497SC.J. Collier *
77497f17497SC.J. Collier * @param priv
77597f17497SC.J. Collier *   Private structure.
77697f17497SC.J. Collier * @param fdir_filter
77797f17497SC.J. Collier *   Flow director filter to find.
77897f17497SC.J. Collier *
77997f17497SC.J. Collier * @return
78097f17497SC.J. Collier *   Filter element if found, otherwise NULL.
78197f17497SC.J. Collier */
78297f17497SC.J. Collierstatic struct mlx5_fdir_filter *
78397f17497SC.J. Collierpriv_find_filter_in_list(struct priv *priv,
78497f17497SC.J. Collier			 const struct rte_eth_fdir_filter *fdir_filter)
78597f17497SC.J. Collier{
78697f17497SC.J. Collier	struct fdir_flow_desc desc;
78797f17497SC.J. Collier	struct mlx5_fdir_filter *mlx5_fdir_filter;
78897f17497SC.J. Collier	enum rte_fdir_mode fdir_mode = priv->dev->data->dev_conf.fdir_conf.mode;
78997f17497SC.J. Collier
79097f17497SC.J. Collier	/* Get flow director filter to look for. */
79197f17497SC.J. Collier	fdir_filter_to_flow_desc(fdir_filter, &desc, fdir_mode);
79297f17497SC.J. Collier
79397f17497SC.J. Collier	/* Look for the requested element. */
79497f17497SC.J. Collier	LIST_FOREACH(mlx5_fdir_filter, priv->fdir_filter_list, next) {
79597f17497SC.J. Collier		/* Only valid elements should be in the list. */
79697f17497SC.J. Collier		assert(mlx5_fdir_filter != NULL);
79797f17497SC.J. Collier
79897f17497SC.J. Collier		/* Return matching filter. */
79997f17497SC.J. Collier		if (!memcmp(&desc, &mlx5_fdir_filter->desc, sizeof(desc)))
80097f17497SC.J. Collier			return mlx5_fdir_filter;
80197f17497SC.J. Collier	}
80297f17497SC.J. Collier
80397f17497SC.J. Collier	/* Filter not found */
80497f17497SC.J. Collier	return NULL;
80597f17497SC.J. Collier}
80697f17497SC.J. Collier
80797f17497SC.J. Collier/**
80897f17497SC.J. Collier * Add new flow director filter and store it in list.
80997f17497SC.J. Collier *
81097f17497SC.J. Collier * @param priv
81197f17497SC.J. Collier *   Private structure.
81297f17497SC.J. Collier * @param fdir_filter
81397f17497SC.J. Collier *   Flow director filter to add.
81497f17497SC.J. Collier *
81597f17497SC.J. Collier * @return
81697f17497SC.J. Collier *   0 on success, errno value on failure.
81797f17497SC.J. Collier */
81897f17497SC.J. Collierstatic int
81997f17497SC.J. Collierpriv_fdir_filter_add(struct priv *priv,
82097f17497SC.J. Collier		     const struct rte_eth_fdir_filter *fdir_filter)
82197f17497SC.J. Collier{
82297f17497SC.J. Collier	struct mlx5_fdir_filter *mlx5_fdir_filter;
82397f17497SC.J. Collier	enum rte_fdir_mode fdir_mode = priv->dev->data->dev_conf.fdir_conf.mode;
82497f17497SC.J. Collier	int err = 0;
82597f17497SC.J. Collier
82697f17497SC.J. Collier	/* Validate queue number. */
82797f17497SC.J. Collier	if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
82897f17497SC.J. Collier		ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
82997f17497SC.J. Collier		return EINVAL;
83097f17497SC.J. Collier	}
83197f17497SC.J. Collier
83297f17497SC.J. Collier	/* Duplicate filters are currently unsupported. */
83397f17497SC.J. Collier	mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
83497f17497SC.J. Collier	if (mlx5_fdir_filter != NULL) {
83597f17497SC.J. Collier		ERROR("filter already exists");
83697f17497SC.J. Collier		return EINVAL;
83797f17497SC.J. Collier	}
83897f17497SC.J. Collier
83997f17497SC.J. Collier	/* Create new flow director filter. */
84097f17497SC.J. Collier	mlx5_fdir_filter =
84197f17497SC.J. Collier		rte_calloc(__func__, 1, sizeof(*mlx5_fdir_filter), 0);
84297f17497SC.J. Collier	if (mlx5_fdir_filter == NULL) {
84397f17497SC.J. Collier		err = ENOMEM;
84497f17497SC.J. Collier		ERROR("cannot allocate flow director filter: %s",
84597f17497SC.J. Collier		      strerror(err));
84697f17497SC.J. Collier		return err;
84797f17497SC.J. Collier	}
84897f17497SC.J. Collier
84932e04ea0SChristian Ehrhardt	/* Set action parameters. */
85097f17497SC.J. Collier	mlx5_fdir_filter->queue = fdir_filter->action.rx_queue;
85132e04ea0SChristian Ehrhardt	mlx5_fdir_filter->behavior = fdir_filter->action.behavior;
85297f17497SC.J. Collier
85397f17497SC.J. Collier	/* Convert to mlx5 filter descriptor. */
85497f17497SC.J. Collier	fdir_filter_to_flow_desc(fdir_filter,
85597f17497SC.J. Collier				 &mlx5_fdir_filter->desc, fdir_mode);
85697f17497SC.J. Collier
85797f17497SC.J. Collier	/* Insert new filter into list. */
85897f17497SC.J. Collier	LIST_INSERT_HEAD(priv->fdir_filter_list, mlx5_fdir_filter, next);
85997f17497SC.J. Collier
86097f17497SC.J. Collier	DEBUG("%p: flow director filter %p added",
86197f17497SC.J. Collier	      (void *)priv, (void *)mlx5_fdir_filter);
86297f17497SC.J. Collier
86397f17497SC.J. Collier	/* Enable filter immediately if device is started. */
86497f17497SC.J. Collier	if (priv->started)
86597f17497SC.J. Collier		err = priv_fdir_filter_enable(priv, mlx5_fdir_filter);
86697f17497SC.J. Collier
86797f17497SC.J. Collier	return err;
86897f17497SC.J. Collier}
86997f17497SC.J. Collier
87097f17497SC.J. Collier/**
87197f17497SC.J. Collier * Update queue for specific filter.
87297f17497SC.J. Collier *
87397f17497SC.J. Collier * @param priv
87497f17497SC.J. Collier *   Private structure.
87597f17497SC.J. Collier * @param fdir_filter
87697f17497SC.J. Collier *   Filter to be updated.
87797f17497SC.J. Collier *
87897f17497SC.J. Collier * @return
87997f17497SC.J. Collier *   0 on success, errno value on failure.
88097f17497SC.J. Collier */
88197f17497SC.J. Collierstatic int
88297f17497SC.J. Collierpriv_fdir_filter_update(struct priv *priv,
88397f17497SC.J. Collier			const struct rte_eth_fdir_filter *fdir_filter)
88497f17497SC.J. Collier{
88597f17497SC.J. Collier	struct mlx5_fdir_filter *mlx5_fdir_filter;
88697f17497SC.J. Collier
88797f17497SC.J. Collier	/* Validate queue number. */
88897f17497SC.J. Collier	if (fdir_filter->action.rx_queue >= priv->rxqs_n) {
88997f17497SC.J. Collier		ERROR("invalid queue number %d", fdir_filter->action.rx_queue);
89097f17497SC.J. Collier		return EINVAL;
89197f17497SC.J. Collier	}
89297f17497SC.J. Collier
89397f17497SC.J. Collier	mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
89497f17497SC.J. Collier	if (mlx5_fdir_filter != NULL) {
89597f17497SC.J. Collier		struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
89697f17497SC.J. Collier		int err = 0;
89797f17497SC.J. Collier
89897f17497SC.J. Collier		/* Update queue number. */
89997f17497SC.J. Collier		mlx5_fdir_filter->queue = fdir_filter->action.rx_queue;
90097f17497SC.J. Collier
90197f17497SC.J. Collier		/* Destroy flow handle. */
90297f17497SC.J. Collier		if (flow != NULL) {
90397f17497SC.J. Collier			claim_zero(ibv_exp_destroy_flow(flow));
90497f17497SC.J. Collier			mlx5_fdir_filter->flow = NULL;
90597f17497SC.J. Collier		}
90697f17497SC.J. Collier		DEBUG("%p: flow director filter %p updated",
90797f17497SC.J. Collier		      (void *)priv, (void *)mlx5_fdir_filter);
90897f17497SC.J. Collier
90997f17497SC.J. Collier		/* Enable filter if device is started. */
91097f17497SC.J. Collier		if (priv->started)
91197f17497SC.J. Collier			err = priv_fdir_filter_enable(priv, mlx5_fdir_filter);
91297f17497SC.J. Collier
91397f17497SC.J. Collier		return err;
91497f17497SC.J. Collier	}
91597f17497SC.J. Collier
91697f17497SC.J. Collier	/* Filter not found, create it. */
91797f17497SC.J. Collier	DEBUG("%p: filter not found for update, creating new filter",
91897f17497SC.J. Collier	      (void *)priv);
91997f17497SC.J. Collier	return priv_fdir_filter_add(priv, fdir_filter);
92097f17497SC.J. Collier}
92197f17497SC.J. Collier
92297f17497SC.J. Collier/**
92397f17497SC.J. Collier * Delete specific filter.
92497f17497SC.J. Collier *
92597f17497SC.J. Collier * @param priv
92697f17497SC.J. Collier *   Private structure.
92797f17497SC.J. Collier * @param fdir_filter
92897f17497SC.J. Collier *   Filter to be deleted.
92997f17497SC.J. Collier *
93097f17497SC.J. Collier * @return
93197f17497SC.J. Collier *   0 on success, errno value on failure.
93297f17497SC.J. Collier */
93397f17497SC.J. Collierstatic int
93497f17497SC.J. Collierpriv_fdir_filter_delete(struct priv *priv,
93597f17497SC.J. Collier			const struct rte_eth_fdir_filter *fdir_filter)
93697f17497SC.J. Collier{
93797f17497SC.J. Collier	struct mlx5_fdir_filter *mlx5_fdir_filter;
93897f17497SC.J. Collier
93997f17497SC.J. Collier	mlx5_fdir_filter = priv_find_filter_in_list(priv, fdir_filter);
94097f17497SC.J. Collier	if (mlx5_fdir_filter != NULL) {
94197f17497SC.J. Collier		struct ibv_exp_flow *flow = mlx5_fdir_filter->flow;
94297f17497SC.J. Collier
94397f17497SC.J. Collier		/* Remove element from list. */
94497f17497SC.J. Collier		LIST_REMOVE(mlx5_fdir_filter, next);
94597f17497SC.J. Collier
94697f17497SC.J. Collier		/* Destroy flow handle. */
94797f17497SC.J. Collier		if (flow != NULL) {
94897f17497SC.J. Collier			claim_zero(ibv_exp_destroy_flow(flow));
94997f17497SC.J. Collier			mlx5_fdir_filter->flow = NULL;
95097f17497SC.J. Collier		}
95197f17497SC.J. Collier
95297f17497SC.J. Collier		DEBUG("%p: flow director filter %p deleted",
95397f17497SC.J. Collier		      (void *)priv, (void *)mlx5_fdir_filter);
95497f17497SC.J. Collier
95597f17497SC.J. Collier		/* Delete filter. */
95697f17497SC.J. Collier		rte_free(mlx5_fdir_filter);
95797f17497SC.J. Collier
95897f17497SC.J. Collier		return 0;
95997f17497SC.J. Collier	}
96097f17497SC.J. Collier
96197f17497SC.J. Collier	ERROR("%p: flow director delete failed, cannot find filter",
96297f17497SC.J. Collier	      (void *)priv);
96397f17497SC.J. Collier	return EINVAL;
96497f17497SC.J. Collier}
96597f17497SC.J. Collier
96697f17497SC.J. Collier/**
96797f17497SC.J. Collier * Get flow director information.
96897f17497SC.J. Collier *
96997f17497SC.J. Collier * @param priv
97097f17497SC.J. Collier *   Private structure.
97197f17497SC.J. Collier * @param[out] fdir_info
97297f17497SC.J. Collier *   Resulting flow director information.
97397f17497SC.J. Collier */
97497f17497SC.J. Collierstatic void
97597f17497SC.J. Collierpriv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info)
97697f17497SC.J. Collier{
97797f17497SC.J. Collier	struct rte_eth_fdir_masks *mask =
97897f17497SC.J. Collier		&priv->dev->data->dev_conf.fdir_conf.mask;
97997f17497SC.J. Collier
98097f17497SC.J. Collier	fdir_info->mode = priv->dev->data->dev_conf.fdir_conf.mode;
98197f17497SC.J. Collier	fdir_info->guarant_spc = 0;
98297f17497SC.J. Collier
98397f17497SC.J. Collier	rte_memcpy(&fdir_info->mask, mask, sizeof(fdir_info->mask));
98497f17497SC.J. Collier
98597f17497SC.J. Collier	fdir_info->max_flexpayload = 0;
98697f17497SC.J. Collier	fdir_info->flow_types_mask[0] = 0;
98797f17497SC.J. Collier
98897f17497SC.J. Collier	fdir_info->flex_payload_unit = 0;
98997f17497SC.J. Collier	fdir_info->max_flex_payload_segment_num = 0;
99097f17497SC.J. Collier	fdir_info->flex_payload_limit = 0;
99197f17497SC.J. Collier	memset(&fdir_info->flex_conf, 0, sizeof(fdir_info->flex_conf));
99297f17497SC.J. Collier}
99397f17497SC.J. Collier
99497f17497SC.J. Collier/**
99597f17497SC.J. Collier * Deal with flow director operations.
99697f17497SC.J. Collier *
99797f17497SC.J. Collier * @param priv
99897f17497SC.J. Collier *   Pointer to private structure.
99997f17497SC.J. Collier * @param filter_op
100097f17497SC.J. Collier *   Operation to perform.
100197f17497SC.J. Collier * @param arg
100297f17497SC.J. Collier *   Pointer to operation-specific structure.
100397f17497SC.J. Collier *
100497f17497SC.J. Collier * @return
100597f17497SC.J. Collier *   0 on success, errno value on failure.
100697f17497SC.J. Collier */
100797f17497SC.J. Collierstatic int
100897f17497SC.J. Collierpriv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg)
100997f17497SC.J. Collier{
101097f17497SC.J. Collier	enum rte_fdir_mode fdir_mode =
101197f17497SC.J. Collier		priv->dev->data->dev_conf.fdir_conf.mode;
101297f17497SC.J. Collier	int ret = 0;
101397f17497SC.J. Collier
101497f17497SC.J. Collier	if (filter_op == RTE_ETH_FILTER_NOP)
101597f17497SC.J. Collier		return 0;
101697f17497SC.J. Collier
101797f17497SC.J. Collier	if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
101897f17497SC.J. Collier	    fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
101997f17497SC.J. Collier		ERROR("%p: flow director mode %d not supported",
102097f17497SC.J. Collier		      (void *)priv, fdir_mode);
102197f17497SC.J. Collier		return EINVAL;
102297f17497SC.J. Collier	}
102397f17497SC.J. Collier
102497f17497SC.J. Collier	switch (filter_op) {
102597f17497SC.J. Collier	case RTE_ETH_FILTER_ADD:
102697f17497SC.J. Collier		ret = priv_fdir_filter_add(priv, arg);
102797f17497SC.J. Collier		break;
102897f17497SC.J. Collier	case RTE_ETH_FILTER_UPDATE:
102997f17497SC.J. Collier		ret = priv_fdir_filter_update(priv, arg);
103097f17497SC.J. Collier		break;
103197f17497SC.J. Collier	case RTE_ETH_FILTER_DELETE:
103297f17497SC.J. Collier		ret = priv_fdir_filter_delete(priv, arg);
103397f17497SC.J. Collier		break;
103497f17497SC.J. Collier	case RTE_ETH_FILTER_FLUSH:
103597f17497SC.J. Collier		priv_fdir_filter_flush(priv);
103697f17497SC.J. Collier		break;
103797f17497SC.J. Collier	case RTE_ETH_FILTER_INFO:
103897f17497SC.J. Collier		priv_fdir_info_get(priv, arg);
103997f17497SC.J. Collier		break;
104097f17497SC.J. Collier	default:
104197f17497SC.J. Collier		DEBUG("%p: unknown operation %u", (void *)priv, filter_op);
104297f17497SC.J. Collier		ret = EINVAL;
104397f17497SC.J. Collier		break;
104497f17497SC.J. Collier	}
104597f17497SC.J. Collier	return ret;
104697f17497SC.J. Collier}
104797f17497SC.J. Collier
104897f17497SC.J. Collier/**
104997f17497SC.J. Collier * Manage filter operations.
105097f17497SC.J. Collier *
105197f17497SC.J. Collier * @param dev
105297f17497SC.J. Collier *   Pointer to Ethernet device structure.
105397f17497SC.J. Collier * @param filter_type
105497f17497SC.J. Collier *   Filter type.