197f17497SC.J. Collier/*-
297f17497SC.J. Collier *   BSD LICENSE
397f17497SC.J. Collier *
497f17497SC.J. Collier *   Copyright 2015 6WIND S.A.
597f17497SC.J. Collier *   Copyright 2015 Mellanox.
697f17497SC.J. Collier *
797f17497SC.J. Collier *   Redistribution and use in source and binary forms, with or without
897f17497SC.J. Collier *   modification, are permitted provided that the following conditions
997f17497SC.J. Collier *   are met:
1097f17497SC.J. Collier *
1197f17497SC.J. Collier *     * Redistributions of source code must retain the above copyright
1297f17497SC.J. Collier *       notice, this list of conditions and the following disclaimer.
1397f17497SC.J. Collier *     * Redistributions in binary form must reproduce the above copyright
1497f17497SC.J. Collier *       notice, this list of conditions and the following disclaimer in
1597f17497SC.J. Collier *       the documentation and/or other materials provided with the
1697f17497SC.J. Collier *       distribution.
1797f17497SC.J. Collier *     * Neither the name of 6WIND S.A. nor the names of its
1897f17497SC.J. Collier *       contributors may be used to endorse or promote products derived
1997f17497SC.J. Collier *       from this software without specific prior written permission.
2097f17497SC.J. Collier *
2197f17497SC.J. Collier *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
2297f17497SC.J. Collier *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
2397f17497SC.J. Collier *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
2497f17497SC.J. Collier *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
2597f17497SC.J. Collier *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
2697f17497SC.J. Collier *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
2797f17497SC.J. Collier *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
2897f17497SC.J. Collier *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
2997f17497SC.J. Collier *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
3097f17497SC.J. Collier *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
3197f17497SC.J. Collier *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
3297f17497SC.J. Collier */
3397f17497SC.J. Collier
3497f17497SC.J. Collier#include <stddef.h>
3597f17497SC.J. Collier#include <assert.h>
3697f17497SC.J. Collier#include <errno.h>
3797f17497SC.J. Collier#include <string.h>
3897f17497SC.J. Collier#include <stdint.h>
3997f17497SC.J. Collier
4097f17497SC.J. Collier/* Verbs header. */
4197f17497SC.J. Collier/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
4297f17497SC.J. Collier#ifdef PEDANTIC
4332e04ea0SChristian Ehrhardt#pragma GCC diagnostic ignored "-Wpedantic"
4497f17497SC.J. Collier#endif
4597f17497SC.J. Collier#include <infiniband/verbs.h>
4697f17497SC.J. Collier#ifdef PEDANTIC
4732e04ea0SChristian Ehrhardt#pragma GCC diagnostic error "-Wpedantic"
4897f17497SC.J. Collier#endif
4997f17497SC.J. Collier
5097f17497SC.J. Collier/* DPDK headers don't like -pedantic. */
5197f17497SC.J. Collier#ifdef PEDANTIC
5232e04ea0SChristian Ehrhardt#pragma GCC diagnostic ignored "-Wpedantic"
5397f17497SC.J. Collier#endif
5497f17497SC.J. Collier#include <rte_mbuf.h>
5597f17497SC.J. Collier#include <rte_malloc.h>
5697f17497SC.J. Collier#include <rte_ethdev.h>
5797f17497SC.J. Collier#include <rte_common.h>
5897f17497SC.J. Collier#ifdef PEDANTIC
5932e04ea0SChristian Ehrhardt#pragma GCC diagnostic error "-Wpedantic"
6097f17497SC.J. Collier#endif
6197f17497SC.J. Collier
6297f17497SC.J. Collier#include "mlx5_utils.h"
638b25d1adSChristian Ehrhardt#include "mlx5_defs.h"
6497f17497SC.J. Collier#include "mlx5.h"
6597f17497SC.J. Collier#include "mlx5_rxtx.h"
6697f17497SC.J. Collier#include "mlx5_autoconf.h"
6797f17497SC.J. Collier#include "mlx5_defs.h"
6897f17497SC.J. Collier
6997f17497SC.J. Collier/**
7097f17497SC.J. Collier * Allocate TX queue elements.
7197f17497SC.J. Collier *
728b25d1adSChristian Ehrhardt * @param txq_ctrl
7397f17497SC.J. Collier *   Pointer to TX queue structure.
7497f17497SC.J. Collier * @param elts_n
7597f17497SC.J. Collier *   Number of elements to allocate.
7697f17497SC.J. Collier */
778b25d1adSChristian Ehrhardtstatic void
788b25d1adSChristian Ehrhardttxq_alloc_elts(struct txq_ctrl *txq_ctrl, unsigned int elts_n)
7997f17497SC.J. Collier{
8097f17497SC.J. Collier	unsigned int i;
8197f17497SC.J. Collier
828b25d1adSChristian Ehrhardt	for (i = 0; (i != elts_n); ++i)
838b25d1adSChristian Ehrhardt		(*txq_ctrl->txq.elts)[i] = NULL;
846b3e017eSChristian Ehrhardt	for (i = 0; (i != (1u << txq_ctrl->txq.wqe_n)); ++i) {
856b3e017eSChristian Ehrhardt		volatile struct mlx5_wqe64 *wqe = &(*txq_ctrl->txq.wqes)[i];
8697f17497SC.J. Collier
878b25d1adSChristian Ehrhardt		memset((void *)(uintptr_t)wqe, 0x0, sizeof(*wqe));
8897f17497SC.J. Collier	}
898b25d1adSChristian Ehrhardt	DEBUG("%p: allocated and configured %u WRs", (void *)txq_ctrl, elts_n);
908b25d1adSChristian Ehrhardt	txq_ctrl->txq.elts_head = 0;
918b25d1adSChristian Ehrhardt	txq_ctrl->txq.elts_tail = 0;
928b25d1adSChristian Ehrhardt	txq_ctrl->txq.elts_comp = 0;
9397f17497SC.J. Collier}
9497f17497SC.J. Collier
9597f17497SC.J. Collier/**
9697f17497SC.J. Collier * Free TX queue elements.
9797f17497SC.J. Collier *
988b25d1adSChristian Ehrhardt * @param txq_ctrl
9997f17497SC.J. Collier *   Pointer to TX queue structure.
10097f17497SC.J. Collier */
10197f17497SC.J. Collierstatic void
1028b25d1adSChristian Ehrhardttxq_free_elts(struct txq_ctrl *txq_ctrl)
10397f17497SC.J. Collier{
1046b3e017eSChristian Ehrhardt	unsigned int elts_n = 1 << txq_ctrl->txq.elts_n;
1058b25d1adSChristian Ehrhardt	unsigned int elts_head = txq_ctrl->txq.elts_head;
1068b25d1adSChristian Ehrhardt	unsigned int elts_tail = txq_ctrl->txq.elts_tail;
1078b25d1adSChristian Ehrhardt	struct rte_mbuf *(*elts)[elts_n] = txq_ctrl->txq.elts;
10897f17497SC.J. Collier
1098b25d1adSChristian Ehrhardt	DEBUG("%p: freeing WRs", (void *)txq_ctrl);
1108b25d1adSChristian Ehrhardt	txq_ctrl->txq.elts_head = 0;
1118b25d1adSChristian Ehrhardt	txq_ctrl->txq.elts_tail = 0;
1128b25d1adSChristian Ehrhardt	txq_ctrl->txq.elts_comp = 0;
11397f17497SC.J. Collier
11497f17497SC.J. Collier	while (elts_tail != elts_head) {
1158b25d1adSChristian Ehrhardt		struct rte_mbuf *elt = (*elts)[elts_tail];
11697f17497SC.J. Collier
1178b25d1adSChristian Ehrhardt		assert(elt != NULL);
118fdd2322bSLuca Boccassi		rte_pktmbuf_free_seg(elt);
11997f17497SC.J. Collier#ifndef NDEBUG
12097f17497SC.J. Collier		/* Poisoning. */
1218b25d1adSChristian Ehrhardt		memset(&(*elts)[elts_tail],
1228b25d1adSChristian Ehrhardt		       0x77,
1238b25d1adSChristian Ehrhardt		       sizeof((*elts)[elts_tail]));
12497f17497SC.J. Collier#endif
12597f17497SC.J. Collier		if (++elts_tail == elts_n)
12697f17497SC.J. Collier			elts_tail = 0;
12797f17497SC.J. Collier	}
12897f17497SC.J. Collier}
12997f17497SC.J. Collier
13097f17497SC.J. Collier/**
13197f17497SC.J. Collier * Clean up a TX queue.
13297f17497SC.J. Collier *
13397f17497SC.J. Collier * Destroy objects, free allocated memory and reset the structure for reuse.
13497f17497SC.J. Collier *
1358b25d1adSChristian Ehrhardt * @param txq_ctrl
13697f17497SC.J. Collier *   Pointer to TX queue structure.
13797f17497SC.J. Collier */
13897f17497SC.J. Colliervoid
1398b25d1adSChristian Ehrhardttxq_cleanup(struct txq_ctrl *txq_ctrl)
14097f17497SC.J. Collier{
14197f17497SC.J. Collier	struct ibv_exp_release_intf_params params;
14297f17497SC.J. Collier	size_t i;
14397f17497SC.J. Collier
1448b25d1adSChristian Ehrhardt	DEBUG("cleaning up %p", (void *)txq_ctrl);
1458b25d1adSChristian Ehrhardt	txq_free_elts(txq_ctrl);
1468b25d1adSChristian Ehrhardt	if (txq_ctrl->if_qp != NULL) {
1478b25d1adSChristian Ehrhardt		assert(txq_ctrl->priv != NULL);
1488b25d1adSChristian Ehrhardt		assert(txq_ctrl->priv->ctx != NULL);
1498b25d1adSChristian Ehrhardt		assert(txq_ctrl->qp != NULL);
15097f17497SC.J. Collier		params = (struct ibv_exp_release_intf_params){
15197f17497SC.J. Collier			.comp_mask = 0,
15297f17497SC.J. Collier		};
1538b25d1adSChristian Ehrhardt		claim_zero(ibv_exp_release_intf(txq_ctrl->priv->ctx,
1548b25d1adSChristian Ehrhardt						txq_ctrl->if_qp,
15597f17497SC.J. Collier						&params));
15697f17497SC.J. Collier	}
1578b25d1adSChristian Ehrhardt	if (txq_ctrl->if_cq != NULL) {
1588b25d1adSChristian Ehrhardt		assert(txq_ctrl->priv != NULL);
1598b25d1adSChristian Ehrhardt		assert(txq_ctrl->priv->ctx != NULL);
1608b25d1adSChristian Ehrhardt		assert(txq_ctrl->cq != NULL);
16197f17497SC.J. Collier		params = (struct ibv_exp_release_intf_params){
16297f17497SC.J. Collier			.comp_mask = 0,
16397f17497SC.J. Collier		};
1648b25d1adSChristian Ehrhardt		claim_zero(ibv_exp_release_intf(txq_ctrl->priv->ctx,
1658b25d1adSChristian Ehrhardt						txq_ctrl->if_cq,
16697f17497SC.J. Collier						&params));
16797f17497SC.J. Collier	}
1688b25d1adSChristian Ehrhardt	if (txq_ctrl->qp != NULL)
1698b25d1adSChristian Ehrhardt		claim_zero(ibv_destroy_qp(txq_ctrl->qp));
1708b25d1adSChristian Ehrhardt	if (txq_ctrl->cq != NULL)
1718b25d1adSChristian Ehrhardt		claim_zero(ibv_destroy_cq(txq_ctrl->cq));
1728b25d1adSChristian Ehrhardt	if (txq_ctrl->rd != NULL) {
17397f17497SC.J. Collier		struct ibv_exp_destroy_res_domain_attr attr = {
17497f17497SC.J. Collier			.comp_mask = 0,
17597f17497SC.J. Collier		};
17697f17497SC.J. Collier
1778b25d1adSChristian Ehrhardt		assert(txq_ctrl->priv != NULL);
1788b25d1adSChristian Ehrhardt		assert(txq_ctrl->priv->ctx != NULL);
1798b25d1adSChristian Ehrhardt		claim_zero(ibv_exp_destroy_res_domain(txq_ctrl->priv->ctx,
1808b25d1adSChristian Ehrhardt						      txq_ctrl->rd,
18197f17497SC.J. Collier						      &attr));
18297f17497SC.J. Collier	}
1838b25d1adSChristian Ehrhardt	for (i = 0; (i != RTE_DIM(txq_ctrl->txq.mp2mr)); ++i) {
1848b25d1adSChristian Ehrhardt		if (txq_ctrl->txq.mp2mr[i].mp == NULL)
18597f17497SC.J. Collier			break;
1868b25d1adSChristian Ehrhardt		assert(txq_ctrl->txq.mp2mr[i].mr != NULL);
1878b25d1adSChristian Ehrhardt		claim_zero(ibv_dereg_mr(txq_ctrl->txq.mp2mr[i].mr));
1888b25d1adSChristian Ehrhardt	}
1898b25d1adSChristian Ehrhardt	memset(txq_ctrl, 0, sizeof(*txq_ctrl));
1908b25d1adSChristian Ehrhardt}
1918b25d1adSChristian Ehrhardt
1928b25d1adSChristian Ehrhardt/**
1938b25d1adSChristian Ehrhardt * Initialize TX queue.
1948b25d1adSChristian Ehrhardt *
1958b25d1adSChristian Ehrhardt * @param tmpl
1968b25d1adSChristian Ehrhardt *   Pointer to TX queue control template.
1978b25d1adSChristian Ehrhardt * @param txq_ctrl
1988b25d1adSChristian Ehrhardt *   Pointer to TX queue control.
1998b25d1adSChristian Ehrhardt *
2008b25d1adSChristian Ehrhardt * @return
2018b25d1adSChristian Ehrhardt *   0 on success, errno value on failure.
2028b25d1adSChristian Ehrhardt */
2038b25d1adSChristian Ehrhardtstatic inline int
2048b25d1adSChristian Ehrhardttxq_setup(struct txq_ctrl *tmpl, struct txq_ctrl *txq_ctrl)
2058b25d1adSChristian Ehrhardt{
2068b25d1adSChristian Ehrhardt	struct mlx5_qp *qp = to_mqp(tmpl->qp);
2078b25d1adSChristian Ehrhardt	struct ibv_cq *ibcq = tmpl->cq;
208fdd2322bSLuca Boccassi	struct ibv_mlx5_cq_info cq_info;
2098b25d1adSChristian Ehrhardt
210fdd2322bSLuca Boccassi	if (ibv_mlx5_exp_get_cq_info(ibcq, &cq_info)) {
211fdd2322bSLuca Boccassi		ERROR("Unable to query CQ info. check your OFED.");
212fdd2322bSLuca Boccassi		return ENOTSUP;
213fdd2322bSLuca Boccassi	}
214fdd2322bSLuca Boccassi	if (cq_info.cqe_size != RTE_CACHE_LINE_SIZE) {
2158b25d1adSChristian Ehrhardt		ERROR("Wrong MLX5_CQE_SIZE environment variable value: "
2168b25d1adSChristian Ehrhardt		      "it should be set to %u", RTE_CACHE_LINE_SIZE);
2178b25d1adSChristian Ehrhardt		return EINVAL;
21897f17497SC.J. Collier	}
219fdd2322bSLuca Boccassi	tmpl->txq.cqe_n = log2above(cq_info.cqe_cnt);
2208b25d1adSChristian Ehrhardt	tmpl->txq.qp_num_8s = qp->ctrl_seg.qp_num << 8;
2218b25d1adSChristian Ehrhardt	tmpl->txq.wqes =
2226b3e017eSChristian Ehrhardt		(volatile struct mlx5_wqe64 (*)[])
2238b25d1adSChristian Ehrhardt		(uintptr_t)qp->gen_data.sqstart;
2246b3e017eSChristian Ehrhardt	tmpl->txq.wqe_n = log2above(qp->sq.wqe_cnt);
2258b25d1adSChristian Ehrhardt	tmpl->txq.qp_db = &qp->gen_data.db[MLX5_SND_DBR];
2268b25d1adSChristian Ehrhardt	tmpl->txq.bf_reg = qp->gen_data.bf->reg;
227fdd2322bSLuca Boccassi	tmpl->txq.cq_db = cq_info.dbrec;
2288b25d1adSChristian Ehrhardt	tmpl->txq.cqes =
2298b25d1adSChristian Ehrhardt		(volatile struct mlx5_cqe (*)[])
230fdd2322bSLuca Boccassi		(uintptr_t)cq_info.buf;
2318b25d1adSChristian Ehrhardt	tmpl->txq.elts =
2326b3e017eSChristian Ehrhardt		(struct rte_mbuf *(*)[1 << tmpl->txq.elts_n])
2338b25d1adSChristian Ehrhardt		((uintptr_t)txq_ctrl + sizeof(*txq_ctrl));
2348b25d1adSChristian Ehrhardt	return 0;
23597f17497SC.J. Collier}
23697f17497SC.J. Collier
23797f17497SC.J. Collier/**
23897f17497SC.J. Collier * Configure a TX queue.
23997f17497SC.J. Collier *
24097f17497SC.J. Collier * @param dev
24197f17497SC.J. Collier *   Pointer to Ethernet device structure.
2428b25d1adSChristian Ehrhardt * @param txq_ctrl
24397f17497SC.J. Collier *   Pointer to TX queue structure.
24497f17497SC.J. Collier * @param desc
24597f17497SC.J. Collier *   Number of descriptors to configure in queue.
24697f17497SC.J. Collier * @param socket
24797f17497SC.J. Collier *   NUMA socket on which memory must be allocated.
24897f17497SC.J. Collier * @param[in] conf
24997f17497SC.J. Collier *   Thresholds parameters.
25097f17497SC.J. Collier *
25197f17497SC.J. Collier * @return
25297f17497SC.J. Collier *   0 on success, errno value on failure.
25397f17497SC.J. Collier */
25497f17497SC.J. Collierint
2558b25d1adSChristian Ehrhardttxq_ctrl_setup(struct rte_eth_dev *dev, struct txq_ctrl *txq_ctrl,
2568b25d1adSChristian Ehrhardt	       uint16_t desc, unsigned int socket,
2578b25d1adSChristian Ehrhardt	       const struct rte_eth_txconf *conf)
25897f17497SC.J. Collier{
25997f17497SC.J. Collier	struct priv *priv = mlx5_get_priv(dev);
2608b25d1adSChristian Ehrhardt	struct txq_ctrl tmpl = {
26197f17497SC.J. Collier		.priv = priv,
2628b25d1adSChristian Ehrhardt		.socket = socket,
26397f17497SC.J. Collier	};
26497f17497SC.J. Collier	union {
26597f17497SC.J. Collier		struct ibv_exp_query_intf_params params;
26697f17497SC.J. Collier		struct ibv_exp_qp_init_attr init;
26797f17497SC.J. Collier		struct ibv_exp_res_domain_init_attr rd;
26897f17497SC.J. Collier		struct ibv_exp_cq_init_attr cq;
26997f17497SC.J. Collier		struct ibv_exp_qp_attr mod;
2708b25d1adSChristian Ehrhardt		struct ibv_exp_cq_attr cq_attr;
27197f17497SC.J. Collier	} attr;
27297f17497SC.J. Collier	enum ibv_exp_query_intf_status status;
27397f17497SC.J. Collier	int ret = 0;
27497f17497SC.J. Collier
2758b25d1adSChristian Ehrhardt	if (mlx5_getenv_int("MLX5_ENABLE_CQE_COMPRESSION")) {
2768b25d1adSChristian Ehrhardt		ret = ENOTSUP;
2778b25d1adSChristian Ehrhardt		ERROR("MLX5_ENABLE_CQE_COMPRESSION must never be set");
2788b25d1adSChristian Ehrhardt		goto error;
27997f17497SC.J. Collier	}
2808b25d1adSChristian Ehrhardt	(void)conf; /* Thresholds configuration (ignored). */
2818b25d1adSChristian Ehrhardt	assert(desc > MLX5_TX_COMP_THRESH);
2826b3e017eSChristian Ehrhardt	tmpl.txq.elts_n = log2above(desc);
28397f17497SC.J. Collier	/* MRs will be registered in mp2mr[] later. */
28497f17497SC.J. Collier	attr.rd = (struct ibv_exp_res_domain_init_attr){
28597f17497SC.J. Collier		.comp_mask = (IBV_EXP_RES_DOMAIN_THREAD_MODEL |
28697f17497SC.J. Collier			      IBV_EXP_RES_DOMAIN_MSG_MODEL),
28797f17497SC.J. Collier		.thread_model = IBV_EXP_THREAD_SINGLE,
28897f17497SC.J. Collier		.msg_model = IBV_EXP_MSG_HIGH_BW,
28997f17497SC.J. Collier	};
29097f17497SC.J. Collier	tmpl.rd = ibv_exp_create_res_domain(priv->ctx, &attr.rd);
29197f17497SC.J. Collier	if (tmpl.rd == NULL) {
29297f17497SC.J. Collier		ret = ENOMEM;
29397f17497SC.J. Collier		ERROR("%p: RD creation failure: %s",
29497f17497SC.J. Collier		      (void *)dev, strerror(ret));
29597f17497SC.J. Collier		goto error;
29697f17497SC.J. Collier	}
29797f17497SC.J. Collier	attr.cq = (struct ibv_exp_cq_init_attr){
29897f17497SC.J. Collier		.comp_mask = IBV_EXP_CQ_INIT_ATTR_RES_DOMAIN,
29997f17497SC.J. Collier		.res_domain = tmpl.rd,
30097f17497SC.J. Collier	};
3018b25d1adSChristian Ehrhardt	tmpl.cq = ibv_exp_create_cq(priv->ctx,
3028b25d1adSChristian Ehrhardt				    (((desc / MLX5_TX_COMP_THRESH) - 1) ?
3038b25d1adSChristian Ehrhardt				     ((desc / MLX5_TX_COMP_THRESH) - 1) : 1),
3048b25d1adSChristian Ehrhardt				    NULL, NULL, 0, &attr.cq);
30597f17497SC.J. Collier	if (tmpl.cq == NULL) {
30697f17497SC.J. Collier		ret = ENOMEM;
30797f17497SC.J. Collier		ERROR("%p: CQ creation failure: %s",
30897f17497SC.J. Collier		      (void *)dev, strerror(ret));
30997f17497SC.J. Collier		goto error;
31097f17497SC.J. Collier	}
31197f17497SC.J. Collier	DEBUG("priv->device_attr.max_qp_wr is %d",
31297f17497SC.J. Collier	      priv->device_attr.max_qp_wr);
31397f17497SC.J. Collier	DEBUG("priv->device_attr.max_sge is %d",
31497f17497SC.J. Collier	      priv->device_attr.max_sge);
31597f17497SC.J. Collier	attr.init = (struct ibv_exp_qp_init_attr){
31697f17497SC.J. Collier		/* CQ to be associated with the send queue. */
31797f17497SC.J. Collier		.send_cq = tmpl.cq,
31897f17497SC.J. Collier		/* CQ to be associated with the receive queue. */
31997f17497SC.J. Collier		.recv_cq = tmpl.cq,
32097f17497SC.J. Collier		.cap = {
32197f17497SC.J. Collier			/* Max number of outstanding WRs. */
32297f17497SC.J. Collier			.max_send_wr = ((priv->device_attr.max_qp_wr < desc) ?
32397f17497SC.J. Collier					priv->device_attr.max_qp_wr :
32497f17497SC.J. Collier					desc),
3258b25d1adSChristian Ehrhardt			/*
3268b25d1adSChristian Ehrhardt			 * Max number of scatter/gather elements in a WR,
3278b25d1adSChristian Ehrhardt			 * must be 1 to prevent libmlx5 from trying to affect
3288b25d1adSChristian Ehrhardt			 * too much memory. TX gather is not impacted by the
3298b25d1adSChristian Ehrhardt			 * priv->device_attr.max_sge limit and will still work
3308b25d1adSChristian Ehrhardt			 * properly.
3318b25d1adSChristian Ehrhardt			 */
3328b25d1adSChristian Ehrhardt			.max_send_sge = 1,
33397f17497SC.J. Collier		},
33497f17497SC.J. Collier		.qp_type = IBV_QPT_RAW_PACKET,
33597f17497SC.J. Collier		/* Do *NOT* enable this, completions events are managed per
33697f17497SC.J. Collier		 * TX burst. */
33797f17497SC.J. Collier		.sq_sig_all = 0,
33897f17497SC.J. Collier		.pd = priv->pd,
33997f17497SC.J. Collier		.res_domain = tmpl.rd,
34097f17497SC.J. Collier		.comp_mask = (IBV_EXP_QP_INIT_ATTR_PD |
34197f17497SC.J. Collier			      IBV_EXP_QP_INIT_ATTR_RES_DOMAIN),
34297f17497SC.J. Collier	};
34332e04ea0SChristian Ehrhardt	if (priv->txq_inline && (priv->txqs_n >= priv->txqs_inline)) {
34432e04ea0SChristian Ehrhardt		tmpl.txq.max_inline =
34532e04ea0SChristian Ehrhardt			((priv->txq_inline + (RTE_CACHE_LINE_SIZE - 1)) /
34632e04ea0SChristian Ehrhardt			 RTE_CACHE_LINE_SIZE);
34732e04ea0SChristian Ehrhardt		attr.init.cap.max_inline_data =
34832e04ea0SChristian Ehrhardt			tmpl.txq.max_inline * RTE_CACHE_LINE_SIZE;
3498b25d1adSChristian Ehrhardt	}
35097f17497SC.J. Collier	tmpl.qp = ibv_exp_create_qp(priv->ctx, &attr.init);
35197f17497SC.J. Collier	if (tmpl.qp == NULL) {
35297f17497SC.J. Collier		ret = (errno ? errno : EINVAL);
35397f17497SC.J. Collier		ERROR("%p: QP creation failure: %s",
35497f17497SC.J. Collier		      (void *)dev, strerror(ret));
35597f17497SC.J. Collier		goto error;
35697f17497SC.J. Collier	}
3578b25d1adSChristian Ehrhardt	DEBUG("TX queue capabilities: max_send_wr=%u, max_send_sge=%u,"
3588b25d1adSChristian Ehrhardt	      " max_inline_data=%u",
3598b25d1adSChristian Ehrhardt	      attr.init.cap.max_send_wr,
3608b25d1adSChristian Ehrhardt	      attr.init.cap.max_send_sge,
3618b25d1adSChristian Ehrhardt	      attr.init.cap.max_inline_data);
36297f17497SC.J. Collier	attr.mod = (struct ibv_exp_qp_attr){
36397f17497SC.J. Collier		/* Move the QP to this state. */
36497f17497SC.J. Collier		.qp_state = IBV_QPS_INIT,
36597f17497SC.J. Collier		/* Primary port number. */
36697f17497SC.J. Collier		.port_num = priv->port
36797f17497SC.J. Collier	};
36897f17497SC.J. Collier	ret = ibv_exp_modify_qp(tmpl.qp, &attr.mod,
36997f17497SC.J. Collier				(IBV_EXP_QP_STATE | IBV_EXP_QP_PORT));
37097f17497SC.J. Collier	if (ret) {
37197f17497SC.J. Collier		ERROR("%p: QP state to IBV_QPS_INIT failed: %s",
37297f17497SC.J. Collier		      (void *)dev, strerror(ret));
37397f17497SC.J. Collier		goto error;
37497f17497SC.J. Collier	}
3758b25d1adSChristian Ehrhardt	ret = txq_setup(&tmpl, txq_ctrl);
37697f17497SC.J. Collier	if (ret) {
3778b25d1adSChristian Ehrhardt		ERROR("%p: cannot initialize TX queue structure: %s",
37897f17497SC.J. Collier		      (void *)dev, strerror(ret));
37997f17497SC.J. Collier		goto error;
38097f17497SC.J. Collier	}
3818b25d1adSChristian Ehrhardt	txq_alloc_elts(&tmpl, desc);
38297f17497SC.J. Collier	attr.mod = (struct ibv_exp_qp_attr){
38397f17497SC.J. Collier		.qp_state = IBV_QPS_RTR
38497f17497SC.J. Collier	};
38597f17497SC.J. Collier	ret = ibv_exp_modify_qp(tmpl.qp, &attr.mod, IBV_EXP_QP_STATE);
38697f17497SC.J. Collier	if (ret) {
38797f17497SC.J. Collier		ERROR("%p: QP state to IBV_QPS_RTR failed: %s",
38897f17497SC.J. Collier		      (void *)dev, strerror(ret));
38997f17497SC.J. Collier		goto error;
39097f17497SC.J. Collier	}
39197f17497SC.J. Collier	attr.mod.qp_state = IBV_QPS_RTS;
39297f17497SC.J. Collier	ret = ibv_exp_modify_qp(tmpl.qp, &attr.mod, IBV_EXP_QP_STATE);
39397f17497SC.J. Collier	if (ret) {
39497f17497SC.J. Collier		ERROR("%p: QP state to IBV_QPS_RTS failed: %s",
39597f17497SC.J. Collier		      (void *)dev, strerror(ret));
39697f17497SC.J. Collier		goto error;
39797f17497SC.J. Collier	}
39897f17497SC.J. Collier	attr.params = (struct ibv_exp_query_intf_params){
39997f17497SC.J. Collier		.intf_scope = IBV_EXP_INTF_GLOBAL,
40097f17497SC.J. Collier		.intf = IBV_EXP_INTF_CQ,
40197f17497SC.J. Collier		.obj = tmpl.cq,
40297f17497SC.J. Collier	};
40397f17497SC.J. Collier	tmpl.if_cq = ibv_exp_query_intf(priv->ctx, &attr.params, &status);
40497f17497SC.J. Collier	if (tmpl.if_cq == NULL) {
40597f17497SC.J. Collier		ret = EINVAL;
40697f17497SC.J. Collier		ERROR("%p: CQ interface family query failed with status %d",
40797f17497SC.J. Collier		      (void *)dev, status);
40897f17497SC.J. Collier		goto error;
40997f17497SC.J. Collier	}
41097f17497SC.J. Collier	attr.params = (struct ibv_exp_query_intf_params){
41197f17497SC.J. Collier		.intf_scope = IBV_EXP_INTF_GLOBAL,
41297f17497SC.J. Collier		.intf = IBV_EXP_INTF_QP_BURST,
41397f17497SC.J. Collier		.intf_version = 1,
4148b25d1adSChristian Ehrhardt		.obj = tmpl.qp,
41597f17497SC.J. Collier		/* Enable multi-packet send if supported. */
41697f17497SC.J. Collier		.family_flags =
4178b25d1adSChristian Ehrhardt			((priv->mps && !priv->sriov) ?
41897f17497SC.J. Collier			 IBV_EXP_QP_BURST_CREATE_ENABLE_MULTI_PACKET_SEND_WR :
41997f17497SC.J. Collier			 0),
42097f17497SC.J. Collier	};
42197f17497SC.J. Collier	tmpl.if_qp = ibv_exp_query_intf(priv->ctx, &attr.params, &status);
42297f17497SC.J. Collier	if (tmpl.if_qp == NULL) {
42397f17497SC.J. Collier		ret = EINVAL;
42497f17497SC.J. Collier		ERROR("%p: QP interface family query failed with status %d",
42597f17497SC.J. Collier		      (void *)dev, status);
42697f17497SC.J. Collier		goto error;
42797f17497SC.J. Collier	}
42897f17497SC.J. Collier	/* Clean up txq in case we're reinitializing it. */
4298b25d1adSChristian Ehrhardt	DEBUG("%p: cleaning-up old txq just in case", (void *)txq_ctrl);
4308b25d1adSChristian Ehrhardt	txq_cleanup(txq_ctrl);
4318b25d1adSChristian Ehrhardt	*txq_ctrl = tmpl;
4328b25d1adSChristian Ehrhardt	DEBUG("%p: txq updated with %p", (void *)txq_ctrl, (void *)&tmpl);
43397f17497SC.J. Collier	/* Pre-register known mempools. */
4348b25d1adSChristian Ehrhardt	rte_mempool_walk(txq_mp2mr_iter, txq_ctrl);
43597f17497SC.J. Collier	assert(ret == 0);
43697f17497SC.J. Collier	return 0;
43797f17497SC.J. Colliererror:
43897f17497SC.J. Collier	txq_cleanup(&tmpl);
43997f17497SC.J. Collier	assert(ret > 0);
44097f17497SC.J. Collier	return ret;
44197f17497SC.J. Collier}
44297f17497SC.J. Collier
44397f17497SC.J. Collier/**
44497f17497SC.J. Collier * DPDK callback to configure a TX queue.
44597f17497SC.J. Collier *
44697f17497SC.J. Collier * @param dev
44797f17497SC.J. Collier *   Pointer to Ethernet device structure.
44897f17497SC.J. Collier * @param idx
44997f17497SC.J. Collier *   TX queue index.
45097f17497SC.J. Collier * @param desc
45197f17497SC.J. Collier *   Number of descriptors to configure in queue.
45297f17497SC.J. Collier * @param socket
45397f17497SC.J. Collier *   NUMA socket on which memory must be allocated.
45497f17497SC.J. Collier * @param[in] conf
45597f17497SC.J. Collier *   Thresholds parameters.
45697f17497SC.J. Collier *
45797f17497SC.J. Collier * @return
45897f17497SC.J. Collier *   0 on success, negative errno value on failure.
45997f17497SC.J. Collier */
46097f17497SC.J. Collierint
46197f17497SC.J. Colliermlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
46297f17497SC.J. Collier		    unsigned int socket, const struct rte_eth_txconf *conf)
46397f17497SC.J. Collier{
46497f17497SC.J. Collier	struct priv *priv = dev->data->dev_private;
46597f17497SC.J. Collier	struct txq *txq = (*priv->txqs)[idx];
4668b25d1adSChristian Ehrhardt	struct txq_ctrl *txq_ctrl = container_of(txq, struct txq_ctrl, txq);
46797f17497SC.J. Collier	int ret;
46897f17497SC.J. Collier
46997f17497SC.J. Collier	if (mlx5_is_secondary())
47097f17497SC.J. Collier		return -E_RTE_SECONDARY;
47197f17497SC.J. Collier
47297f17497SC.J. Collier	priv_lock(priv);
4738b25d1adSChristian Ehrhardt	if (desc <= MLX5_TX_COMP_THRESH) {
4748b25d1adSChristian Ehrhardt		WARN("%p: number of descriptors requested for TX queue %u"
4758b25d1adSChristian Ehrhardt		     " must be higher than MLX5_TX_COMP_THRESH, using"
4768b25d1adSChristian Ehrhardt		     " %u instead of %u",
4778b25d1adSChristian Ehrhardt		     (void *)dev, idx, MLX5_TX_COMP_THRESH + 1, desc);
4788b25d1adSChristian Ehrhardt		desc = MLX5_TX_COMP_THRESH + 1;
4798b25d1adSChristian Ehrhardt	}
4808b25d1adSChristian Ehrhardt	if (!rte_is_power_of_2(desc)) {
4818b25d1adSChristian Ehrhardt		desc = 1 << log2above(desc);
4828b25d1adSChristian Ehrhardt		WARN("%p: increased number of descriptors in TX queue %u"
4838b25d1adSChristian Ehrhardt		     " to the next power of two (%d)",
4848b25d1adSChristian Ehrhardt		     (void *)dev, idx, desc);
4858b25d1adSChristian Ehrhardt	}
48697f17497SC.J. Collier	DEBUG("%p: configuring queue %u for %u descriptors",
48797f17497SC.J. Collier	      (void *)dev, idx, desc);
48897f17497SC.J. Collier	if (idx >= priv->txqs_n) {
48997f17497SC.J. Collier		ERROR("%p: queue index out of range (%u >= %u)",
49097f17497SC.J. Collier		      (void *)dev, idx, priv->txqs_n);
49197f17497SC.J. Collier		priv_unlock(priv);
49297f17497SC.J. Collier		return -EOVERFLOW;
49397f17497SC.J. Collier	}
49497f17497SC.J. Collier	if (txq != NULL) {
49597f17497SC.J. Collier		DEBUG("%p: reusing already allocated queue index %u (%p)",
49697f17497SC.J. Collier		      (void *)dev, idx, (void *)txq);
49797f17497SC.J. Collier		if (priv->started) {
49897f17497SC.J. Collier			priv_unlock(priv);
49997f17497SC.J. Collier			return -EEXIST;
50097f17497SC.J. Collier		}
50197f17497SC.J. Collier		(*priv->txqs)[idx] = NULL;
5028b25d1adSChristian Ehrhardt		txq_cleanup(txq_ctrl);
503aab0c291SChristian Ehrhardt		/* Resize if txq size is changed. */
504aab0c291SChristian Ehrhardt		if (txq_ctrl->txq.elts_n != log2above(desc)) {
505aab0c291SChristian Ehrhardt			txq_ctrl = rte_realloc(txq_ctrl,
506aab0c291SChristian Ehrhardt					       sizeof(*txq_ctrl) +
507aab0c291SChristian Ehrhardt					       desc * sizeof(struct rte_mbuf *),
508aab0c291SChristian Ehrhardt					       RTE_CACHE_LINE_SIZE);
509aab0c291SChristian Ehrhardt			if (!txq_ctrl) {
510aab0c291SChristian Ehrhardt				ERROR("%p: unable to reallocate queue index %u",
511aab0c291SChristian Ehrhardt					(void *)dev, idx);
512aab0c291SChristian Ehrhardt				priv_unlock(priv);
513aab0c291SChristian Ehrhardt				return -ENOMEM;
514aab0c291SChristian Ehrhardt			}
515aab0c291SChristian Ehrhardt		}
51697f17497SC.J. Collier	} else {
5178b25d1adSChristian Ehrhardt		txq_ctrl =
5188b25d1adSChristian Ehrhardt			rte_calloc_socket("TXQ", 1,
5198b25d1adSChristian Ehrhardt					  sizeof(*txq_ctrl) +
5208b25d1adSChristian Ehrhardt					  desc * sizeof(struct rte_mbuf *),
5218b25d1adSChristian Ehrhardt					  0, socket);
5228b25d1adSChristian Ehrhardt		if (txq_ctrl == NULL) {
52397f17497SC.J. Collier			ERROR("%p: unable to allocate queue index %u",
52497f17497SC.J. Collier			      (void *)dev, idx);
52597f17497SC.J. Collier			priv_unlock(priv);
52697f17497SC.J. Collier			return -ENOMEM;
52797f17497SC.J. Collier		}
52897f17497SC.J. Collier	}
5298b25d1adSChristian Ehrhardt	ret = txq_ctrl_setup(dev, txq_ctrl, desc, socket, conf);
53097f17497SC.J. Collier	if (ret)
5318b25d1adSChristian Ehrhardt		rte_free(txq_ctrl);
53297f17497SC.J. Collier	else {
5338b25d1adSChristian Ehrhardt		txq_ctrl->txq.stats.idx = idx;
53497f17497SC.J. Collier		DEBUG("%p: adding TX queue %p to list",
5358b25d1adSChristian Ehrhardt		      (void *)dev, (void *)txq_ctrl);
5368b25d1adSChristian Ehrhardt		(*priv->txqs)[idx] = &txq_ctrl->txq;
53797f17497SC.J. Collier		/* Update send callback. */
5388b25d1adSChristian Ehrhardt		priv_select_tx_function(priv);
53997f17497SC.J. Collier	}
54097f17497SC.J. Collier	priv_unlock(priv);
54197f17497SC.J. Collier	return -ret;
54297f17497SC.J. Collier}
54397f17497SC.J. Collier
54497f17497SC.J. Collier/**
54597f17497SC.J. Collier * DPDK callback to release a TX queue.
54697f17497SC.J. Collier *
54797f17497SC.J. Collier * @param dpdk_txq
54897f17497SC.J. Collier *   Generic TX queue pointer.
54997f17497SC.J. Collier */
55097f17497SC.J. Colliervoid
55197f17497SC.J. Colliermlx5_tx_queue_release(void *dpdk_txq)
55297f17497SC.J. Collier{
55397f17497SC.J. Collier	struct txq *txq = (struct txq *)dpdk_txq;
5548b25d1adSChristian Ehrhardt	struct txq_ctrl *txq_ctrl;
55597f17497SC.J. Collier	struct priv *priv;
55697f17497SC.J. Collier	unsigned int i;
55797f17497SC.J. Collier
55897f17497SC.J. Collier	if (mlx5_is_secondary())
55997f17497SC.J. Collier		return;
56097f17497SC.J. Collier
56197f17497SC.J. Collier	if (txq == NULL)
56297f17497SC.J. Collier		return;
5638b25d1adSChristian Ehrhardt	txq_ctrl = container_of(txq, struct txq_ctrl, txq);
5648b25d1adSChristian Ehrhardt	priv = txq_ctrl->priv;
56597f17497SC.J. Collier	priv_lock(priv);
56697f17497SC.J. Collier	for (i = 0; (i != priv->txqs_n); ++i)
56797f17497SC.J. Collier		if ((*priv->txqs)[i] == txq) {
56897f17497SC.J. Collier			DEBUG("%p: removing TX queue %p from list",
5698b25d1adSChristian Ehrhardt			      (void *)priv->dev, (void *)txq_ctrl);
57097f17497SC.J. Collier			(*priv->txqs)[i] = NULL;
57197f17497SC.J. Collier			break;
57297f17497SC.J. Collier		}
5738b25d1adSChristian Ehrhardt	txq_cleanup(txq_ctrl);
5748b25d1adSChristian Ehrhardt	rte_free(txq_ctrl);
57597f17497SC.J. Collier	priv_unlock(priv);
57697f17497SC.J. Collier}
57797f17497SC.J. Collier
57897f17497SC.J. Collier/**
57997f17497SC.J. Collier * DPDK callback for TX in secondary processes.
58097f17497SC.J. Collier *
58197f17497SC.J. Collier * This function configures all queues from primary process information
58297f17497SC.J. Collier * if necessary before reverting to the normal TX burst callback.
58397f17497SC.J. Collier *
58497f17497SC.J. Collier * @param dpdk_txq
58597f17497SC.J. Collier *   Generic pointer to TX queue structure.
58697f17497SC.J. Collier * @param[in] pkts
58797f17497SC.J. Collier *   Packets to transmit.
58897f17497SC.J. Collier * @param pkts_n
58997f17497SC.J. Collier *   Number of packets in array.
59097f17497SC.J. Collier *
59197f17497SC.J. Collier * @return
59297f17497SC.J. Collier *   Number of packets successfully transmitted (<= pkts_n).
59397f17497SC.J. Collier */
59497f17497SC.J. Collieruint16_t
59597f17497SC.J. Colliermlx5_tx_burst_secondary_setup(void *dpdk_txq, struct rte_mbuf **pkts,
59697f17497SC.J. Collier			      uint16_t pkts_n)
59797f17497SC.J. Collier{
59897f17497SC.J. Collier	struct txq *txq = dpdk_txq;
5998b25d1adSChristian Ehrhardt	struct txq_ctrl *txq_ctrl = container_of(txq, struct txq_ctrl, txq);
6008b25d1adSChristian Ehrhardt	struct priv *priv = mlx5_secondary_data_setup(txq_ctrl->priv);
60197f17497SC.J. Collier	struct priv *primary_priv;
60297f17497SC.J. Collier	unsigned int index;
60397f17497SC.J. Collier
60497f17497SC.J. Collier	if (priv == NULL)
60597f17497SC.J. Collier		return 0;
60697f17497SC.J. Collier	primary_priv =
60797f17497SC.J. Collier		mlx5_secondary_data[priv->dev->data->port_id].primary_priv;
60897f17497SC.J. Collier	/* Look for queue index in both private structures. */
60997f17497SC.J. Collier	for (index = 0; index != priv->txqs_n; ++index)
61097f17497SC.J. Collier		if (((*primary_priv->txqs)[index] == txq) ||
61197f17497SC.J. Collier		    ((*priv->txqs)[index] == txq))
61297f17497SC.J. Collier			break;
61397f17497SC.J. Collier	if (index == priv->txqs_n)
61497f17497SC.J. Collier		return 0;
61597f17497SC.J. Collier	txq = (*priv->txqs)[index];
61697f17497SC.J. Collier	return priv->dev->tx_pkt_burst(txq, pkts, pkts_n);
61797f17497SC.J. Collier}
618