15129044dSC.J. Collier/*-
25129044dSC.J. Collier *   BSD LICENSE
35129044dSC.J. Collier *
45129044dSC.J. Collier *   Copyright 2015 6WIND S.A.
55129044dSC.J. Collier *   Copyright 2015 Mellanox.
65129044dSC.J. Collier *
75129044dSC.J. Collier *   Redistribution and use in source and binary forms, with or without
85129044dSC.J. Collier *   modification, are permitted provided that the following conditions
95129044dSC.J. Collier *   are met:
105129044dSC.J. Collier *
115129044dSC.J. Collier *     * Redistributions of source code must retain the above copyright
125129044dSC.J. Collier *       notice, this list of conditions and the following disclaimer.
135129044dSC.J. Collier *     * Redistributions in binary form must reproduce the above copyright
145129044dSC.J. Collier *       notice, this list of conditions and the following disclaimer in
155129044dSC.J. Collier *       the documentation and/or other materials provided with the
165129044dSC.J. Collier *       distribution.
175129044dSC.J. Collier *     * Neither the name of 6WIND S.A. nor the names of its
185129044dSC.J. Collier *       contributors may be used to endorse or promote products derived
195129044dSC.J. Collier *       from this software without specific prior written permission.
205129044dSC.J. Collier *
215129044dSC.J. Collier *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
225129044dSC.J. Collier *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
235129044dSC.J. Collier *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
245129044dSC.J. Collier *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
255129044dSC.J. Collier *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
265129044dSC.J. Collier *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
275129044dSC.J. Collier *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
285129044dSC.J. Collier *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
295129044dSC.J. Collier *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
305129044dSC.J. Collier *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
315129044dSC.J. Collier *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
325129044dSC.J. Collier */
335129044dSC.J. Collier
345129044dSC.J. Collier#include <stddef.h>
355129044dSC.J. Collier#include <stdint.h>
365129044dSC.J. Collier#include <errno.h>
375129044dSC.J. Collier#include <string.h>
385129044dSC.J. Collier#include <assert.h>
395129044dSC.J. Collier
405129044dSC.J. Collier/* Verbs header. */
415129044dSC.J. Collier/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
425129044dSC.J. Collier#ifdef PEDANTIC
43c300e355SChristian Ehrhardt#pragma GCC diagnostic ignored "-Wpedantic"
445129044dSC.J. Collier#endif
455129044dSC.J. Collier#include <infiniband/verbs.h>
465129044dSC.J. Collier#ifdef PEDANTIC
47c300e355SChristian Ehrhardt#pragma GCC diagnostic error "-Wpedantic"
485129044dSC.J. Collier#endif
495129044dSC.J. Collier
505129044dSC.J. Collier/* DPDK headers don't like -pedantic. */
515129044dSC.J. Collier#ifdef PEDANTIC
52c300e355SChristian Ehrhardt#pragma GCC diagnostic ignored "-Wpedantic"
535129044dSC.J. Collier#endif
545129044dSC.J. Collier#include <rte_malloc.h>
555129044dSC.J. Collier#include <rte_ethdev.h>
565129044dSC.J. Collier#ifdef PEDANTIC
57c300e355SChristian Ehrhardt#pragma GCC diagnostic error "-Wpedantic"
585129044dSC.J. Collier#endif
595129044dSC.J. Collier
605129044dSC.J. Collier#include "mlx5.h"
615129044dSC.J. Collier#include "mlx5_rxtx.h"
625129044dSC.J. Collier
635129044dSC.J. Collier/**
645129044dSC.J. Collier * Get a RSS configuration hash key.
655129044dSC.J. Collier *
665129044dSC.J. Collier * @param priv
675129044dSC.J. Collier *   Pointer to private structure.
685129044dSC.J. Collier * @param rss_hf
695129044dSC.J. Collier *   RSS hash functions configuration must be retrieved for.
705129044dSC.J. Collier *
715129044dSC.J. Collier * @return
725129044dSC.J. Collier *   Pointer to a RSS configuration structure or NULL if rss_hf cannot
735129044dSC.J. Collier *   be matched.
745129044dSC.J. Collier */
755129044dSC.J. Collierstatic struct rte_eth_rss_conf *
765129044dSC.J. Collierrss_hash_get(struct priv *priv, uint64_t rss_hf)
775129044dSC.J. Collier{
785129044dSC.J. Collier	unsigned int i;
795129044dSC.J. Collier
805129044dSC.J. Collier	for (i = 0; (i != hash_rxq_init_n); ++i) {
815129044dSC.J. Collier		uint64_t dpdk_rss_hf = hash_rxq_init[i].dpdk_rss_hf;
825129044dSC.J. Collier
835129044dSC.J. Collier		if (!(dpdk_rss_hf & rss_hf))
845129044dSC.J. Collier			continue;
855129044dSC.J. Collier		return (*priv->rss_conf)[i];
865129044dSC.J. Collier	}
875129044dSC.J. Collier	return NULL;
885129044dSC.J. Collier}
895129044dSC.J. Collier
905129044dSC.J. Collier/**
915129044dSC.J. Collier * Register a RSS key.
925129044dSC.J. Collier *
935129044dSC.J. Collier * @param priv
945129044dSC.J. Collier *   Pointer to private structure.
955129044dSC.J. Collier * @param key
965129044dSC.J. Collier *   Hash key to register.
975129044dSC.J. Collier * @param key_len
985129044dSC.J. Collier *   Hash key length in bytes.
995129044dSC.J. Collier * @param rss_hf
1005129044dSC.J. Collier *   RSS hash functions the provided key applies to.
1015129044dSC.J. Collier *
1025129044dSC.J. Collier * @return
1035129044dSC.J. Collier *   0 on success, errno value on failure.
1045129044dSC.J. Collier */
1055129044dSC.J. Collierint
1065129044dSC.J. Collierrss_hash_rss_conf_new_key(struct priv *priv, const uint8_t *key,
1075129044dSC.J. Collier			  unsigned int key_len, uint64_t rss_hf)
1085129044dSC.J. Collier{
1095129044dSC.J. Collier	unsigned int i;
1105129044dSC.J. Collier
1115129044dSC.J. Collier	for (i = 0; (i != hash_rxq_init_n); ++i) {
1125129044dSC.J. Collier		struct rte_eth_rss_conf *rss_conf;
1135129044dSC.J. Collier		uint64_t dpdk_rss_hf = hash_rxq_init[i].dpdk_rss_hf;
1145129044dSC.J. Collier
1155129044dSC.J. Collier		if (!(dpdk_rss_hf & rss_hf))
1165129044dSC.J. Collier			continue;
1175129044dSC.J. Collier		rss_conf = rte_realloc((*priv->rss_conf)[i],
1185129044dSC.J. Collier				       (sizeof(*rss_conf) + key_len),
1195129044dSC.J. Collier				       0);
1205129044dSC.J. Collier		if (!rss_conf)
1215129044dSC.J. Collier			return ENOMEM;
1225129044dSC.J. Collier		rss_conf->rss_key = (void *)(rss_conf + 1);
1235129044dSC.J. Collier		rss_conf->rss_key_len = key_len;
1245129044dSC.J. Collier		rss_conf->rss_hf = dpdk_rss_hf;
1255129044dSC.J. Collier		memcpy(rss_conf->rss_key, key, key_len);
1265129044dSC.J. Collier		(*priv->rss_conf)[i] = rss_conf;
1275129044dSC.J. Collier	}
1285129044dSC.J. Collier	return 0;
1295129044dSC.J. Collier}
1305129044dSC.J. Collier
1315129044dSC.J. Collier/**
1325129044dSC.J. Collier * DPDK callback to update the RSS hash configuration.
1335129044dSC.J. Collier *
1345129044dSC.J. Collier * @param dev
1355129044dSC.J. Collier *   Pointer to Ethernet device structure.
1365129044dSC.J. Collier * @param[in] rss_conf
1375129044dSC.J. Collier *   RSS configuration data.
1385129044dSC.J. Collier *
1395129044dSC.J. Collier * @return
1405129044dSC.J. Collier *   0 on success, negative errno value on failure.
1415129044dSC.J. Collier */
1425129044dSC.J. Collierint
1435129044dSC.J. Colliermlx5_rss_hash_update(struct rte_eth_dev *dev,
1445129044dSC.J. Collier		     struct rte_eth_rss_conf *rss_conf)
1455129044dSC.J. Collier{
1465129044dSC.J. Collier	struct priv *priv = dev->data->dev_private;
1475129044dSC.J. Collier	int err = 0;
1485129044dSC.J. Collier
1495129044dSC.J. Collier	priv_lock(priv);
1505129044dSC.J. Collier
1515129044dSC.J. Collier	assert(priv->rss_conf != NULL);
1525129044dSC.J. Collier
1535129044dSC.J. Collier	/* Apply configuration. */
1545129044dSC.J. Collier	if (rss_conf->rss_key)
1555129044dSC.J. Collier		err = rss_hash_rss_conf_new_key(priv,
1565129044dSC.J. Collier						rss_conf->rss_key,
1575129044dSC.J. Collier						rss_conf->rss_key_len,
1585129044dSC.J. Collier						rss_conf->rss_hf);
1595129044dSC.J. Collier	/* Store protocols for which RSS is enabled. */
1605129044dSC.J. Collier	priv->rss_hf = rss_conf->rss_hf;
1615129044dSC.J. Collier	priv_unlock(priv);
1625129044dSC.J. Collier	assert(err >= 0);
1635129044dSC.J. Collier	return -err;
1645129044dSC.J. Collier}
1655129044dSC.J. Collier
1665129044dSC.J. Collier/**
1675129044dSC.J. Collier * DPDK callback to get the RSS hash configuration.
1685129044dSC.J. Collier *
1695129044dSC.J. Collier * @param dev
1705129044dSC.J. Collier *   Pointer to Ethernet device structure.
1715129044dSC.J. Collier * @param[in, out] rss_conf
1725129044dSC.J. Collier *   RSS configuration data.
1735129044dSC.J. Collier *
1745129044dSC.J. Collier * @return
1755129044dSC.J. Collier *   0 on success, negative errno value on failure.
1765129044dSC.J. Collier */
1775129044dSC.J. Collierint
1785129044dSC.J. Colliermlx5_rss_hash_conf_get(struct rte_eth_dev *dev,
1795129044dSC.J. Collier		       struct rte_eth_rss_conf *rss_conf)
1805129044dSC.J. Collier{
1815129044dSC.J. Collier	struct priv *priv = dev->data->dev_private;
1825129044dSC.J. Collier	struct rte_eth_rss_conf *priv_rss_conf;
1835129044dSC.J. Collier
1845129044dSC.J. Collier	priv_lock(priv);
1855129044dSC.J. Collier
1865129044dSC.J. Collier	assert(priv->rss_conf != NULL);
1875129044dSC.J. Collier
1885129044dSC.J. Collier	priv_rss_conf = rss_hash_get(priv, rss_conf->rss_hf);
1895129044dSC.J. Collier	if (!priv_rss_conf) {
1905129044dSC.J. Collier		rss_conf->rss_hf = 0;
1915129044dSC.J. Collier		priv_unlock(priv);
1925129044dSC.J. Collier		return -EINVAL;
1935129044dSC.J. Collier	}
1945129044dSC.J. Collier	if (rss_conf->rss_key &&
1955129044dSC.J. Collier	    rss_conf->rss_key_len >= priv_rss_conf->rss_key_len)
1965129044dSC.J. Collier		memcpy(rss_conf->rss_key,
1975129044dSC.J. Collier		       priv_rss_conf->rss_key,
1985129044dSC.J. Collier		       priv_rss_conf->rss_key_len);
1995129044dSC.J. Collier	rss_conf->rss_key_len = priv_rss_conf->rss_key_len;
2005129044dSC.J. Collier	rss_conf->rss_hf = priv_rss_conf->rss_hf;
2015129044dSC.J. Collier
2025129044dSC.J. Collier	priv_unlock(priv);
2035129044dSC.J. Collier	return 0;
2045129044dSC.J. Collier}
2055129044dSC.J. Collier
2065129044dSC.J. Collier/**
2075129044dSC.J. Collier * Allocate/reallocate RETA index table.
2085129044dSC.J. Collier *
2095129044dSC.J. Collier * @param priv
2105129044dSC.J. Collier *   Pointer to private structure.
2115129044dSC.J. Collier * @praram reta_size
2125129044dSC.J. Collier *   The size of the array to allocate.
2135129044dSC.J. Collier *
2145129044dSC.J. Collier * @return
2155129044dSC.J. Collier *   0 on success, errno value on failure.
2165129044dSC.J. Collier */
2175129044dSC.J. Collierint
2185129044dSC.J. Collierpriv_rss_reta_index_resize(struct priv *priv, unsigned int reta_size)
2195129044dSC.J. Collier{
2205129044dSC.J. Collier	void *mem;
2215129044dSC.J. Collier	unsigned int old_size = priv->reta_idx_n;
2225129044dSC.J. Collier
2235129044dSC.J. Collier	if (priv->reta_idx_n == reta_size)
2245129044dSC.J. Collier		return 0;
2255129044dSC.J. Collier
2265129044dSC.J. Collier	mem = rte_realloc(priv->reta_idx,
2275129044dSC.J. Collier			  reta_size * sizeof((*priv->reta_idx)[0]), 0);
2285129044dSC.J. Collier	if (!mem)
2295129044dSC.J. Collier		return ENOMEM;
2305129044dSC.J. Collier	priv->reta_idx = mem;
2315129044dSC.J. Collier	priv->reta_idx_n = reta_size;
2325129044dSC.J. Collier
2335129044dSC.J. Collier	if (old_size < reta_size)
2345129044dSC.J. Collier		memset(&(*priv->reta_idx)[old_size], 0,
2355129044dSC.J. Collier		       (reta_size - old_size) *
2365129044dSC.J. Collier		       sizeof((*priv->reta_idx)[0]));
2375129044dSC.J. Collier	return 0;
2385129044dSC.J. Collier}
2395129044dSC.J. Collier
2405129044dSC.J. Collier/**
2415129044dSC.J. Collier * Query RETA table.
2425129044dSC.J. Collier *
2435129044dSC.J. Collier * @param priv
2445129044dSC.J. Collier *   Pointer to private structure.
2455129044dSC.J. Collier * @param[in, out] reta_conf
2465129044dSC.J. Collier *   Pointer to the first RETA configuration structure.
2475129044dSC.J. Collier * @param reta_size
2485129044dSC.J. Collier *   Number of entries.
2495129044dSC.J. Collier *
2505129044dSC.J. Collier * @return
2515129044dSC.J. Collier *   0 on success, errno value on failure.
2525129044dSC.J. Collier */
2535129044dSC.J. Collierstatic int
2545129044dSC.J. Collierpriv_dev_rss_reta_query(struct priv *priv,
2555129044dSC.J. Collier			struct rte_eth_rss_reta_entry64 *reta_conf,
2565129044dSC.J. Collier			unsigned int reta_size)
2575129044dSC.J. Collier{
2585129044dSC.J. Collier	unsigned int idx;
2595129044dSC.J. Collier	unsigned int i;
2605129044dSC.J. Collier	int ret;
2615129044dSC.J. Collier
2625129044dSC.J. Collier	/* See RETA comment in mlx5_dev_infos_get(). */
2635129044dSC.J. Collier	ret = priv_rss_reta_index_resize(priv, priv->ind_table_max_size);
2645129044dSC.J. Collier	if (ret)
2655129044dSC.J. Collier		return ret;
2665129044dSC.J. Collier
2675129044dSC.J. Collier	/* Fill each entry of the table even if its bit is not set. */
2685129044dSC.J. Collier	for (idx = 0, i = 0; (i != reta_size); ++i) {
2695129044dSC.J. Collier		idx = i / RTE_RETA_GROUP_SIZE;
2705129044dSC.J. Collier		reta_conf[idx].reta[i % RTE_RETA_GROUP_SIZE] =
2715129044dSC.J. Collier			(*priv->reta_idx)[i];
2725129044dSC.J. Collier	}
2735129044dSC.J. Collier	return 0;
2745129044dSC.J. Collier}
2755129044dSC.J. Collier
2765129044dSC.J. Collier/**
2775129044dSC.J. Collier * Update RETA table.
2785129044dSC.J. Collier *
2795129044dSC.J. Collier * @param priv
2805129044dSC.J. Collier *   Pointer to private structure.
2815129044dSC.J. Collier * @param[in] reta_conf
2825129044dSC.J. Collier *   Pointer to the first RETA configuration structure.
2835129044dSC.J. Collier * @param reta_size
2845129044dSC.J. Collier *   Number of entries.
2855129044dSC.J. Collier *
2865129044dSC.J. Collier * @return
2875129044dSC.J. Collier *   0 on success, errno value on failure.
2885129044dSC.J. Collier */
2895129044dSC.J. Collierstatic int
2905129044dSC.J. Collierpriv_dev_rss_reta_update(struct priv *priv,
2915129044dSC.J. Collier			 struct rte_eth_rss_reta_entry64 *reta_conf,
2925129044dSC.J. Collier			 unsigned int reta_size)
2935129044dSC.J. Collier{
2945129044dSC.J. Collier	unsigned int idx;
2955129044dSC.J. Collier	unsigned int i;
2965129044dSC.J. Collier	unsigned int pos;
2975129044dSC.J. Collier	int ret;
2985129044dSC.J. Collier
2995129044dSC.J. Collier	/* See RETA comment in mlx5_dev_infos_get(). */
3005129044dSC.J. Collier	ret = priv_rss_reta_index_resize(priv, priv->ind_table_max_size);
3015129044dSC.J. Collier	if (ret)
3025129044dSC.J. Collier		return ret;
3035129044dSC.J. Collier
3045129044dSC.J. Collier	for (idx = 0, i = 0; (i != reta_size); ++i) {
3055129044dSC.J. Collier		idx = i / RTE_RETA_GROUP_SIZE;
3065129044dSC.J. Collier		pos = i % RTE_RETA_GROUP_SIZE;
3075129044dSC.J. Collier		if (((reta_conf[idx].mask >> i) & 0x1) == 0)
3085129044dSC.J. Collier			continue;
3095129044dSC.J. Collier		assert(reta_conf[idx].reta[pos] < priv->rxqs_n);
3105129044dSC.J. Collier		(*priv->reta_idx)[i] = reta_conf[idx].reta[pos];
3115129044dSC.J. Collier	}
3125129044dSC.J. Collier	return 0;
3135129044dSC.J. Collier}
3145129044dSC.J. Collier
3155129044dSC.J. Collier/**
3165129044dSC.J. Collier * DPDK callback to get the RETA indirection table.
3175129044dSC.J. Collier *
3185129044dSC.J. Collier * @param dev
3195129044dSC.J. Collier *   Pointer to Ethernet device structure.
3205129044dSC.J. Collier * @param reta_conf
3215129044dSC.J. Collier *   Pointer to RETA configuration structure array.
3225129044dSC.J. Collier * @param reta_size
3235129044dSC.J. Collier *   Size of the RETA table.
3245129044dSC.J. Collier *
3255129044dSC.J. Collier * @return
3265129044dSC.J. Collier *   0 on success, negative errno value on failure.
3275129044dSC.J. Collier */
3285129044dSC.J. Collierint
3295129044dSC.J. Colliermlx5_dev_rss_reta_query(struct rte_eth_dev *dev,
3305129044dSC.J. Collier			struct rte_eth_rss_reta_entry64 *reta_conf,
3315129044dSC.J. Collier			uint16_t reta_size)
3325129044dSC.J. Collier{
3335129044dSC.J. Collier	int ret;
3345129044dSC.J. Collier	struct priv *priv = dev->data->dev_private;
3355129044dSC.J. Collier
3365129044dSC.J. Collier	priv_lock(priv);
3375129044dSC.J. Collier	ret = priv_dev_rss_reta_query(priv, reta_conf, reta_size);
3385129044dSC.J. Collier	priv_unlock(priv);
3395129044dSC.J. Collier	return -ret;
3405129044dSC.J. Collier}
3415129044dSC.J. Collier
3425129044dSC.J. Collier/**
3435129044dSC.J. Collier * DPDK callback to update the RETA indirection table.
3445129044dSC.J. Collier *
3455129044dSC.J. Collier * @param dev
3465129044dSC.J. Collier *   Pointer to Ethernet device structure.
3475129044dSC.J. Collier * @param reta_conf
3485129044dSC.J. Collier *   Pointer to RETA configuration structure array.
3495129044dSC.J. Collier * @param reta_size
3505129044dSC.J. Collier *   Size of the RETA table.
3515129044dSC.J. Collier *
3525129044dSC.J. Collier * @return
3535129044dSC.J. Collier *   0 on success, negative errno value on failure.
3545129044dSC.J. Collier */
3555129044dSC.J. Collierint
3565129044dSC.J. Colliermlx5_dev_rss_reta_update(struct rte_eth_dev *dev,
3575129044dSC.J. Collier			 struct rte_eth_rss_reta_entry64 *reta_conf,
3585129044dSC.J. Collier			 uint16_t reta_size)
3595129044dSC.J. Collier{
3605129044dSC.J. Collier	int ret;
3615129044dSC.J. Collier	struct priv *priv = dev->data->dev_private;
3625129044dSC.J. Collier
3635129044dSC.J. Collier	priv_lock(priv);
3645129044dSC.J. Collier	ret = priv_dev_rss_reta_update(priv, reta_conf, reta_size);
3655129044dSC.J. Collier	priv_unlock(priv);
3665129044dSC.J. Collier	return -ret;
3675129044dSC.J. Collier}
368