197f17497SC.J. Collier/*-
297f17497SC.J. Collier *   BSD LICENSE
397f17497SC.J. Collier *
497f17497SC.J. Collier *   Copyright 2015 6WIND S.A.
597f17497SC.J. Collier *   Copyright 2015 Mellanox.
697f17497SC.J. Collier *
797f17497SC.J. Collier *   Redistribution and use in source and binary forms, with or without
897f17497SC.J. Collier *   modification, are permitted provided that the following conditions
997f17497SC.J. Collier *   are met:
1097f17497SC.J. Collier *
1197f17497SC.J. Collier *     * Redistributions of source code must retain the above copyright
1297f17497SC.J. Collier *       notice, this list of conditions and the following disclaimer.
1397f17497SC.J. Collier *     * Redistributions in binary form must reproduce the above copyright
1497f17497SC.J. Collier *       notice, this list of conditions and the following disclaimer in
1597f17497SC.J. Collier *       the documentation and/or other materials provided with the
1697f17497SC.J. Collier *       distribution.
1797f17497SC.J. Collier *     * Neither the name of 6WIND S.A. nor the names of its
1897f17497SC.J. Collier *       contributors may be used to endorse or promote products derived
1997f17497SC.J. Collier *       from this software without specific prior written permission.
2097f17497SC.J. Collier *
3297f17497SC.J. Collier */
3397f17497SC.J. Collier
3497f17497SC.J. Collier#include <assert.h>
3597f17497SC.J. Collier#include <stdint.h>
3697f17497SC.J. Collier#include <string.h>
3797f17497SC.J. Collier#include <stdlib.h>
3897f17497SC.J. Collier
3997f17497SC.J. Collier/* Verbs header. */
4097f17497SC.J. Collier/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
4197f17497SC.J. Collier#ifdef PEDANTIC
4232e04ea0SChristian Ehrhardt#pragma GCC diagnostic ignored "-Wpedantic"
4397f17497SC.J. Collier#endif
4497f17497SC.J. Collier#include <infiniband/verbs.h>
458b25d1adSChristian Ehrhardt#include <infiniband/mlx5_hw.h>
468b25d1adSChristian Ehrhardt#include <infiniband/arch.h>
4797f17497SC.J. Collier#ifdef PEDANTIC
4832e04ea0SChristian Ehrhardt#pragma GCC diagnostic error "-Wpedantic"
4997f17497SC.J. Collier#endif
5097f17497SC.J. Collier
5197f17497SC.J. Collier/* DPDK headers don't like -pedantic. */
5297f17497SC.J. Collier#ifdef PEDANTIC
5332e04ea0SChristian Ehrhardt#pragma GCC diagnostic ignored "-Wpedantic"
5497f17497SC.J. Collier#endif
5597f17497SC.J. Collier#include <rte_mbuf.h>
5697f17497SC.J. Collier#include <rte_mempool.h>
5797f17497SC.J. Collier#include <rte_prefetch.h>
5897f17497SC.J. Collier#include <rte_common.h>
5997f17497SC.J. Collier#include <rte_branch_prediction.h>
608b25d1adSChristian Ehrhardt#include <rte_ether.h>
6197f17497SC.J. Collier#ifdef PEDANTIC
6232e04ea0SChristian Ehrhardt#pragma GCC diagnostic error "-Wpedantic"
6397f17497SC.J. Collier#endif
6497f17497SC.J. Collier
6597f17497SC.J. Collier#include "mlx5.h"
6697f17497SC.J. Collier#include "mlx5_utils.h"
6797f17497SC.J. Collier#include "mlx5_rxtx.h"
6897f17497SC.J. Collier#include "mlx5_autoconf.h"
6997f17497SC.J. Collier#include "mlx5_defs.h"
708b25d1adSChristian Ehrhardt#include "mlx5_prm.h"
718b25d1adSChristian Ehrhardt
72ce3d555eSChristian Ehrhardtstatic inline int
73ce3d555eSChristian Ehrhardtcheck_cqe(volatile struct mlx5_cqe *cqe,
74ce3d555eSChristian Ehrhardt	  unsigned int cqes_n, const uint16_t ci)
75ce3d555eSChristian Ehrhardt	  __attribute__((always_inline));
76ce3d555eSChristian Ehrhardt
77ce3d555eSChristian Ehrhardtstatic inline uint32_t
78ce3d555eSChristian Ehrhardttxq_mp2mr(struct txq *txq, struct rte_mempool *mp)
79ce3d555eSChristian Ehrhardt	__attribute__((always_inline));
80ce3d555eSChristian Ehrhardt
81ce3d555eSChristian Ehrhardtstatic inline void
82ce3d555eSChristian Ehrhardtmlx5_tx_dbrec(struct txq *txq, volatile struct mlx5_wqe *wqe)
83ce3d555eSChristian Ehrhardt	__attribute__((always_inline));
84ce3d555eSChristian Ehrhardt
85ce3d555eSChristian Ehrhardtstatic inline uint32_t
86ce3d555eSChristian Ehrhardtrxq_cq_to_pkt_type(volatile struct mlx5_cqe *cqe)
87ce3d555eSChristian Ehrhardt	__attribute__((always_inline));
88ce3d555eSChristian Ehrhardt
89ce3d555eSChristian Ehrhardtstatic inline int
90ce3d555eSChristian Ehrhardtmlx5_rx_poll_len(struct rxq *rxq, volatile struct mlx5_cqe *cqe,
91ce3d555eSChristian Ehrhardt		 uint16_t cqe_cnt, uint32_t *rss_hash)
92ce3d555eSChristian Ehrhardt		 __attribute__((always_inline));
93ce3d555eSChristian Ehrhardt
94ce3d555eSChristian Ehrhardtstatic inline uint32_t
95ce3d555eSChristian Ehrhardtrxq_cq_to_ol_flags(struct rxq *rxq, volatile struct mlx5_cqe *cqe)
96ce3d555eSChristian Ehrhardt		   __attribute__((always_inline));
97ce3d555eSChristian Ehrhardt
988b25d1adSChristian Ehrhardt#ifndef NDEBUG
9997f17497SC.J. Collier
10097f17497SC.J. Collier/**
1018b25d1adSChristian Ehrhardt * Verify or set magic value in CQE.
10297f17497SC.J. Collier *
1038b25d1adSChristian Ehrhardt * @param cqe
1048b25d1adSChristian Ehrhardt *   Pointer to CQE.
10597f17497SC.J. Collier *
10697f17497SC.J. Collier * @return
1078b25d1adSChristian Ehrhardt *   0 the first time.
10897f17497SC.J. Collier */
1098b25d1adSChristian Ehrhardtstatic inline int
1106b3e017eSChristian Ehrhardtcheck_cqe_seen(volatile struct mlx5_cqe *cqe)
11197f17497SC.J. Collier{
1128b25d1adSChristian Ehrhardt	static const uint8_t magic[] = "seen";
1136b3e017eSChristian Ehrhardt	volatile uint8_t (*buf)[sizeof(cqe->rsvd3)] = &cqe->rsvd3;
1148b25d1adSChristian Ehrhardt	int ret = 1;
1158b25d1adSChristian Ehrhardt	unsigned int i;
11697f17497SC.J. Collier
1178b25d1adSChristian Ehrhardt	for (i = 0; i < sizeof(magic) && i < sizeof(*buf); ++i)
1188b25d1adSChristian Ehrhardt		if (!ret || (*buf)[i] != magic[i]) {
1198b25d1adSChristian Ehrhardt			ret = 0;
1208b25d1adSChristian Ehrhardt			(*buf)[i] = magic[i];
1218b25d1adSChristian Ehrhardt		}
1228b25d1adSChristian Ehrhardt	return ret;
1238b25d1adSChristian Ehrhardt}
12497f17497SC.J. Collier
1258b25d1adSChristian Ehrhardt#endif /* NDEBUG */
1268b25d1adSChristian Ehrhardt
1278b25d1adSChristian Ehrhardt/**
1288b25d1adSChristian Ehrhardt * Check whether CQE is valid.
1298b25d1adSChristian Ehrhardt *
1308b25d1adSChristian Ehrhardt * @param cqe
1318b25d1adSChristian Ehrhardt *   Pointer to CQE.
1328b25d1adSChristian Ehrhardt * @param cqes_n
1338b25d1adSChristian Ehrhardt *   Size of completion queue.
1348b25d1adSChristian Ehrhardt * @param ci
1358b25d1adSChristian Ehrhardt *   Consumer index.
1368b25d1adSChristian Ehrhardt *
1378b25d1adSChristian Ehrhardt * @return
1388b25d1adSChristian Ehrhardt *   0 on success, 1 on failure.
1398b25d1adSChristian Ehrhardt */
1408b25d1adSChristian Ehrhardtstatic inline int
1416b3e017eSChristian Ehrhardtcheck_cqe(volatile struct mlx5_cqe *cqe,
1426b3e017eSChristian Ehrhardt	  unsigned int cqes_n, const uint16_t ci)
1438b25d1adSChristian Ehrhardt{
1448b25d1adSChristian Ehrhardt	uint16_t idx = ci & cqes_n;
1458b25d1adSChristian Ehrhardt	uint8_t op_own = cqe->op_own;
1468b25d1adSChristian Ehrhardt	uint8_t op_owner = MLX5_CQE_OWNER(op_own);
1478b25d1adSChristian Ehrhardt	uint8_t op_code = MLX5_CQE_OPCODE(op_own);
14897f17497SC.J. Collier
1498b25d1adSChristian Ehrhardt	if (unlikely((op_owner != (!!(idx))) || (op_code == MLX5_CQE_INVALID)))
1508b25d1adSChristian Ehrhardt		return 1; /* No CQE. */
15197f17497SC.J. Collier#ifndef NDEBUG
1528b25d1adSChristian Ehrhardt	if ((op_code == MLX5_CQE_RESP_ERR) ||
1538b25d1adSChristian Ehrhardt	    (op_code == MLX5_CQE_REQ_ERR)) {
1548b25d1adSChristian Ehrhardt		volatile struct mlx5_err_cqe *err_cqe = (volatile void *)cqe;
1558b25d1adSChristian Ehrhardt		uint8_t syndrome = err_cqe->syndrome;
15697f17497SC.J. Collier
1578b25d1adSChristian Ehrhardt		if ((syndrome == MLX5_CQE_SYNDROME_LOCAL_LENGTH_ERR) ||
1588b25d1adSChristian Ehrhardt		    (syndrome == MLX5_CQE_SYNDROME_REMOTE_ABORTED_ERR))
1598b25d1adSChristian Ehrhardt			return 0;
1606b3e017eSChristian Ehrhardt		if (!check_cqe_seen(cqe))
1618b25d1adSChristian Ehrhardt			ERROR("unexpected CQE error %u (0x%02x)"
1628b25d1adSChristian Ehrhardt			      " syndrome 0x%02x",
1638b25d1adSChristian Ehrhardt			      op_code, op_code, syndrome);
1648b25d1adSChristian Ehrhardt		return 1;
1658b25d1adSChristian Ehrhardt	} else if ((op_code != MLX5_CQE_RESP_SEND) &&
1668b25d1adSChristian Ehrhardt		   (op_code != MLX5_CQE_REQ)) {
1676b3e017eSChristian Ehrhardt		if (!check_cqe_seen(cqe))
1688b25d1adSChristian Ehrhardt			ERROR("unexpected CQE opcode %u (0x%02x)",
1698b25d1adSChristian Ehrhardt			      op_code, op_code);
1708b25d1adSChristian Ehrhardt		return 1;
17197f17497SC.J. Collier	}
1728b25d1adSChristian Ehrhardt#endif /* NDEBUG */
17397f17497SC.J. Collier	return 0;
17497f17497SC.J. Collier}
17597f17497SC.J. Collier
1766b3e017eSChristian Ehrhardtstatic inline void
1776b3e017eSChristian Ehrhardttxq_complete(struct txq *txq) __attribute__((always_inline));
1786b3e017eSChristian Ehrhardt
17997f17497SC.J. Collier/**
1808b25d1adSChristian Ehrhardt * Manage TX completions.
18197f17497SC.J. Collier *
1828b25d1adSChristian Ehrhardt * When sending a burst, mlx5_tx_burst() posts several WRs.
18397f17497SC.J. Collier *
1848b25d1adSChristian Ehrhardt * @param txq
1858b25d1adSChristian Ehrhardt *   Pointer to TX queue structure.
18697f17497SC.J. Collier */
1876b3e017eSChristian Ehrhardtstatic inline void
1888b25d1adSChristian Ehrhardttxq_complete(struct txq *txq)
18997f17497SC.J. Collier{
1906b3e017eSChristian Ehrhardt	const unsigned int elts_n = 1 << txq->elts_n;
1916b3e017eSChristian Ehrhardt	const unsigned int cqe_n = 1 << txq->cqe_n;
1928b25d1adSChristian Ehrhardt	const unsigned int cqe_cnt = cqe_n - 1;
1938b25d1adSChristian Ehrhardt	uint16_t elts_free = txq->elts_tail;
1948b25d1adSChristian Ehrhardt	uint16_t elts_tail;
1958b25d1adSChristian Ehrhardt	uint16_t cq_ci = txq->cq_ci;
1966b3e017eSChristian Ehrhardt	volatile struct mlx5_cqe *cqe = NULL;
1976b3e017eSChristian Ehrhardt	volatile struct mlx5_wqe *wqe;
1988b25d1adSChristian Ehrhardt
1998b25d1adSChristian Ehrhardt	do {
2006b3e017eSChristian Ehrhardt		volatile struct mlx5_cqe *tmp;
20197f17497SC.J. Collier
2026b3e017eSChristian Ehrhardt		tmp = &(*txq->cqes)[cq_ci & cqe_cnt];
2036b3e017eSChristian Ehrhardt		if (check_cqe(tmp, cqe_n, cq_ci))
2048b25d1adSChristian Ehrhardt			break;
2058b25d1adSChristian Ehrhardt		cqe = tmp;
2068b25d1adSChristian Ehrhardt#ifndef NDEBUG
2078b25d1adSChristian Ehrhardt		if (MLX5_CQE_FORMAT(cqe->op_own) == MLX5_COMPRESSED) {
2086b3e017eSChristian Ehrhardt			if (!check_cqe_seen(cqe))
2098b25d1adSChristian Ehrhardt				ERROR("unexpected compressed CQE, TX stopped");
2108b25d1adSChristian Ehrhardt			return;
2118b25d1adSChristian Ehrhardt		}
2128b25d1adSChristian Ehrhardt		if ((MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_RESP_ERR) ||
2138b25d1adSChristian Ehrhardt		    (MLX5_CQE_OPCODE(cqe->op_own) == MLX5_CQE_REQ_ERR)) {
2146b3e017eSChristian Ehrhardt			if (!check_cqe_seen(cqe))
2158b25d1adSChristian Ehrhardt				ERROR("unexpected error CQE, TX stopped");
2168b25d1adSChristian Ehrhardt			return;
2178b25d1adSChristian Ehrhardt		}
2188b25d1adSChristian Ehrhardt#endif /* NDEBUG */
2198b25d1adSChristian Ehrhardt		++cq_ci;
2208b25d1adSChristian Ehrhardt	} while (1);
2218b25d1adSChristian Ehrhardt	if (unlikely(cqe == NULL))
2228b25d1adSChristian Ehrhardt		return;
223ce3d555eSChristian Ehrhardt	wqe = &(*txq->wqes)[ntohs(cqe->wqe_counter) &
2246b3e017eSChristian Ehrhardt			    ((1 << txq->wqe_n) - 1)].hdr;
2256b3e017eSChristian Ehrhardt	elts_tail = wqe->ctrl[3];
2266b3e017eSChristian Ehrhardt	assert(elts_tail < (1 << txq->wqe_n));
2278b25d1adSChristian Ehrhardt	/* Free buffers. */
2288b25d1adSChristian Ehrhardt	while (elts_free != elts_tail) {
2298b25d1adSChristian Ehrhardt		struct rte_mbuf *elt = (*txq->elts)[elts_free];
2308b25d1adSChristian Ehrhardt		unsigned int elts_free_next =
2318b25d1adSChristian Ehrhardt			(elts_free + 1) & (elts_n - 1);
2328b25d1adSChristian Ehrhardt		struct rte_mbuf *elt_next = (*txq->elts)[elts_free_next];
2338b25d1adSChristian Ehrhardt
2348b25d1adSChristian Ehrhardt#ifndef NDEBUG
2358b25d1adSChristian Ehrhardt		/* Poisoning. */
2368b25d1adSChristian Ehrhardt		memset(&(*txq->elts)[elts_free],
2378b25d1adSChristian Ehrhardt		       0x66,
2388b25d1adSChristian Ehrhardt		       sizeof((*txq->elts)[elts_free]));
2398b25d1adSChristian Ehrhardt#endif
2408b25d1adSChristian Ehrhardt		RTE_MBUF_PREFETCH_TO_FREE(elt_next);
2418b25d1adSChristian Ehrhardt		/* Only one segment needs to be freed. */
2428b25d1adSChristian Ehrhardt		rte_pktmbuf_free_seg(elt);
2438b25d1adSChristian Ehrhardt		elts_free = elts_free_next;
24497f17497SC.J. Collier	}
2458b25d1adSChristian Ehrhardt	txq->cq_ci = cq_ci;
2468b25d1adSChristian Ehrhardt	txq->elts_tail = elts_tail;
2478b25d1adSChristian Ehrhardt	/* Update the consumer index. */
2488b25d1adSChristian Ehrhardt	rte_wmb();
2498b25d1adSChristian Ehrhardt	*txq->cq_db = htonl(cq_ci);
25097f17497SC.J. Collier}
25197f17497SC.J. Collier
25297f17497SC.J. Collier/**
25397f17497SC.J. Collier * Get Memory Pool (MP) from mbuf. If mbuf is indirect, the pool from which
25497f17497SC.J. Collier * the cloned mbuf is allocated is returned instead.
25597f17497SC.J. Collier *
25697f17497SC.J. Collier * @param buf
25797f17497SC.J. Collier *   Pointer to mbuf.
25897f17497SC.J. Collier *
25997f17497SC.J. Collier * @return
26097f17497SC.J. Collier *   Memory pool where data is located for given mbuf.
26197f17497SC.J. Collier */
26297f17497SC.J. Collierstatic struct rte_mempool *
26397f17497SC.J. Colliertxq_mb2mp(struct rte_mbuf *buf)
26497f17497SC.J. Collier{
26597f17497SC.J. Collier	if (unlikely(RTE_MBUF_INDIRECT(buf)))
26697f17497SC.J. Collier		return rte_mbuf_from_indirect(buf)->pool;
26797f17497SC.J. Collier	return buf->pool;
26897f17497SC.J. Collier}
26997f17497SC.J. Collier
27097f17497SC.J. Collier/**
27197f17497SC.J. Collier * Get Memory Region (MR) <-> Memory Pool (MP) association from txq->mp2mr[].
27297f17497SC.J. Collier * Add MP to txq->mp2mr[] if it's not registered yet. If mp2mr[] is full,
27397f17497SC.J. Collier * remove an entry first.
27497f17497SC.J. Collier *
27597f17497SC.J. Collier * @param txq
27697f17497SC.J. Collier *   Pointer to TX queue structure.
27797f17497SC.J. Collier * @param[in] mp
27897f17497SC.J. Collier *   Memory Pool for which a Memory Region lkey must be returned.
27997f17497SC.J. Collier *
28097f17497SC.J. Collier * @return
28197f17497SC.J. Collier *   mr->lkey on success, (uint32_t)-1 on failure.
28297f17497SC.J. Collier */
2838b25d1adSChristian Ehrhardtstatic inline uint32_t
2848b25d1adSChristian Ehrhardttxq_mp2mr(struct txq *txq, struct rte_mempool *mp)
28597f17497SC.J. Collier{
28697f17497SC.J. Collier	unsigned int i;
2878b25d1adSChristian Ehrhardt	uint32_t lkey = (uint32_t)-1;
28897f17497SC.J. Collier
28997f17497SC.J. Collier	for (i = 0; (i != RTE_DIM(txq->mp2mr)); ++i) {
29097f17497SC.J. Collier		if (unlikely(txq->mp2mr[i].mp == NULL)) {
29197f17497SC.J. Collier			/* Unknown MP, add a new MR for it. */
29297f17497SC.J. Collier			break;
29397f17497SC.J. Collier		}
29497f17497SC.J. Collier		if (txq->mp2mr[i].mp == mp) {
29597f17497SC.J. Collier			assert(txq->mp2mr[i].lkey != (uint32_t)-1);
2968b25d1adSChristian Ehrhardt			assert(htonl(txq->mp2mr[i].mr->lkey) ==
2978b25d1adSChristian Ehrhardt			       txq->mp2mr[i].lkey);
2988b25d1adSChristian Ehrhardt			lkey = txq->mp2mr[i].lkey;
2998b25d1adSChristian Ehrhardt			break;
30097f17497SC.J. Collier		}
30197f17497SC.J. Collier	}
3028b25d1adSChristian Ehrhardt	if (unlikely(lkey == (uint32_t)-1))
3038b25d1adSChristian Ehrhardt		lkey = txq_mp2mr_reg(txq, mp, i);
3048b25d1adSChristian Ehrhardt	return lkey;
30597f17497SC.J. Collier}
30697f17497SC.J. Collier
3078b25d1adSChristian Ehrhardt/**
3088b25d1adSChristian Ehrhardt * Ring TX queue doorbell.
3098b25d1adSChristian Ehrhardt *
3108b25d1adSChristian Ehrhardt * @param txq
3118b25d1adSChristian Ehrhardt *   Pointer to TX queue structure.
312ce3d555eSChristian Ehrhardt * @param wqe
313ce3d555eSChristian Ehrhardt *   Pointer to the last WQE posted in the NIC.
3148b25d1adSChristian Ehrhardt */
3158b25d1adSChristian Ehrhardtstatic inline void
316ce3d555eSChristian Ehrhardtmlx5_tx_dbrec(struct txq *txq, volatile struct mlx5_wqe *wqe)
3178b25d1adSChristian Ehrhardt{
318ce3d555eSChristian Ehrhardt	uint64_t *dst = (uint64_t *)((uintptr_t)txq->bf_reg);
319ce3d555eSChristian Ehrhardt	volatile uint64_t *src = ((volatile uint64_t *)wqe);
320ce3d555eSChristian Ehrhardt
3218b25d1adSChristian Ehrhardt	rte_wmb();
3228b25d1adSChristian Ehrhardt	*txq->qp_db = htonl(txq->wqe_ci);
3238b25d1adSChristian Ehrhardt	/* Ensure ordering between DB record and BF copy. */
3248b25d1adSChristian Ehrhardt	rte_wmb();
325ce3d555eSChristian Ehrhardt	*dst = *src;
3268b25d1adSChristian Ehrhardt}
32797f17497SC.J. Collier
3288b25d1adSChristian Ehrhardt/**
3298b25d1adSChristian Ehrhardt * Prefetch a CQE.
3308b25d1adSChristian Ehrhardt *
3318b25d1adSChristian Ehrhardt * @param txq
3328b25d1adSChristian Ehrhardt *   Pointer to TX queue structure.
3338b25d1adSChristian Ehrhardt * @param cqe_ci
3348b25d1adSChristian Ehrhardt *   CQE consumer index.
3358b25d1adSChristian Ehrhardt */
3368b25d1adSChristian Ehrhardtstatic inline void
3378b25d1adSChristian Ehrhardttx_prefetch_cqe(struct txq *txq, uint16_t ci)
3388b25d1adSChristian Ehrhardt{
3396b3e017eSChristian Ehrhardt	volatile struct mlx5_cqe *cqe;
34097f17497SC.J. Collier
3416b3e017eSChristian Ehrhardt	cqe = &(*txq->cqes)[ci & ((1 << txq->cqe_n) - 1)];
3428b25d1adSChristian Ehrhardt	rte_prefetch0(cqe);
34397f17497SC.J. Collier}
34497f17497SC.J. Collier
3458b25d1adSChristian Ehrhardt/**
3468b25d1adSChristian Ehrhardt * Prefetch a WQE.
3478b25d1adSChristian Ehrhardt *
3488b25d1adSChristian Ehrhardt * @param txq
3498b25d1adSChristian Ehrhardt *   Pointer to TX queue structure.
3508b25d1adSChristian Ehrhardt * @param  wqe_ci
3518b25d1adSChristian Ehrhardt *   WQE consumer index.
3528b25d1adSChristian Ehrhardt */
3538b25d1adSChristian Ehrhardtstatic inline void
3548b25d1adSChristian Ehrhardttx_prefetch_wqe(struct txq *txq, uint16_t ci)
3558b25d1adSChristian Ehrhardt{
3566b3e017eSChristian Ehrhardt	volatile struct mlx5_wqe64 *wqe;
3578b25d1adSChristian Ehrhardt
3586b3e017eSChristian Ehrhardt	wqe = &(*txq->wqes)[ci & ((1 << txq->wqe_n) - 1)];
3598b25d1adSChristian Ehrhardt	rte_prefetch0(wqe);
3608b25d1adSChristian Ehrhardt}
36197f17497SC.J. Collier
36297f17497SC.J. Collier/**
3638b25d1adSChristian Ehrhardt * DPDK callback for TX.
36497f17497SC.J. Collier *
3658b25d1adSChristian Ehrhardt * @param dpdk_txq
3668b25d1adSChristian Ehrhardt *   Generic pointer to TX queue structure.
3678b25d1adSChristian Ehrhardt * @param[in] pkts
3688b25d1adSChristian Ehrhardt *   Packets to transmit.
3698b25d1adSChristian Ehrhardt * @param pkts_n
3708b25d1adSChristian Ehrhardt *   Number of packets in array.
37197f17497SC.J. Collier *
37297f17497SC.J. Collier * @return
3738b25d1adSChristian Ehrhardt *   Number of packets successfully transmitted (<= pkts_n).
37497f17497SC.J. Collier */
3758b25d1adSChristian Ehrhardtuint16_t
3768b25d1adSChristian Ehrhardtmlx5_tx_burst(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
37797f17497SC.J. Collier{
3788b25d1adSChristian Ehrhardt	struct txq *txq = (struct txq *)dpdk_txq;
3798b25d1adSChristian Ehrhardt	uint16_t elts_head = txq->elts_head;
3806b3e017eSChristian Ehrhardt	const unsigned int elts_n = 1 << txq->elts_n;
3818b25d1adSChristian Ehrhardt	unsigned int i = 0;
3828b25d1adSChristian Ehrhardt	unsigned int j = 0;
3838b25d1adSChristian Ehrhardt	unsigned int max;
3848b25d1adSChristian Ehrhardt	unsigned int comp;
3856b3e017eSChristian Ehrhardt	volatile struct mlx5_wqe *wqe = NULL;
3866b3e017eSChristian Ehrhardt	unsigned int segs_n = 0;
3876b3e017eSChristian Ehrhardt	struct rte_mbuf *buf = NULL;
3886b3e017eSChristian Ehrhardt	uint8_t *raw;
38997f17497SC.J. Collier
3908b25d1adSChristian Ehrhardt	if (unlikely(!pkts_n))
3918b25d1adSChristian Ehrhardt		return 0;
3928b25d1adSChristian Ehrhardt	/* Prefetch first packet cacheline. */
3938b25d1adSChristian Ehrhardt	tx_prefetch_cqe(txq, txq->cq_ci);
3948b25d1adSChristian Ehrhardt	tx_prefetch_cqe(txq, txq->cq_ci + 1);
3958b25d1adSChristian Ehrhardt	rte_prefetch0(*pkts);
3968b25d1adSChristian Ehrhardt	/* Start processing. */
3978b25d1adSChristian Ehrhardt	txq_complete(txq);
3988b25d1adSChristian Ehrhardt	max = (elts_n - (elts_head - txq->elts_tail));
3998b25d1adSChristian Ehrhardt	if (max > elts_n)
4008b25d1adSChristian Ehrhardt		max -= elts_n;
40197f17497SC.J. Collier	do {
4026b3e017eSChristian Ehrhardt		volatile struct mlx5_wqe_data_seg *dseg = NULL;
4038b25d1adSChristian Ehrhardt		uint32_t length;
4046b3e017eSChristian Ehrhardt		unsigned int ds = 0;
4056b3e017eSChristian Ehrhardt		uintptr_t addr;
4066b3e017eSChristian Ehrhardt#ifdef MLX5_PMD_SOFT_COUNTERS
4076b3e017eSChristian Ehrhardt		uint32_t total_length = 0;
4086b3e017eSChristian Ehrhardt#endif
40997f17497SC.J. Collier
4106b3e017eSChristian Ehrhardt		/* first_seg */
4116b3e017eSChristian Ehrhardt		buf = *(pkts++);
4126b3e017eSChristian Ehrhardt		segs_n = buf->nb_segs;
4138b25d1adSChristian Ehrhardt		/*
4148b25d1adSChristian Ehrhardt		 * Make sure there is enough room to store this packet and
4158b25d1adSChristian Ehrhardt		 * that one ring entry remains unused.
4168b25d1adSChristian Ehrhardt		 */
4178b25d1adSChristian Ehrhardt		assert(segs_n);
4188b25d1adSChristian Ehrhardt		if (max < segs_n + 1)
4198b25d1adSChristian Ehrhardt			break;
4208b25d1adSChristian Ehrhardt		max -= segs_n;
4216b3e017eSChristian Ehrhardt		--segs_n;
4226b3e017eSChristian Ehrhardt		if (!segs_n)
4236b3e017eSChristian Ehrhardt			--pkts_n;
4246b3e017eSChristian Ehrhardt		wqe = &(*txq->wqes)[txq->wqe_ci &
4256b3e017eSChristian Ehrhardt				    ((1 << txq->wqe_n) - 1)].hdr;
42632e04ea0SChristian Ehrhardt		tx_prefetch_wqe(txq, txq->wqe_ci + 1);
4276b3e017eSChristian Ehrhardt		if (pkts_n > 1)
4288b25d1adSChristian Ehrhardt			rte_prefetch0(*pkts);
4296b3e017eSChristian Ehrhardt		addr = rte_pktmbuf_mtod(buf, uintptr_t);
4308b25d1adSChristian Ehrhardt		length = DATA_LEN(buf);
4316b3e017eSChristian Ehrhardt#ifdef MLX5_PMD_SOFT_COUNTERS
4326b3e017eSChristian Ehrhardt		total_length = length;
4336b3e017eSChristian Ehrhardt#endif
43447d9763aSLuca Boccassi		if (length < (MLX5_WQE_DWORD_SIZE + 2)) {
43547d9763aSLuca Boccassi			txq->stats.oerrors++;
436aab0c291SChristian Ehrhardt			break;
43747d9763aSLuca Boccassi		}
4388b25d1adSChristian Ehrhardt		/* Update element. */
4398b25d1adSChristian Ehrhardt		(*txq->elts)[elts_head] = buf;
4406b3e017eSChristian Ehrhardt		elts_head = (elts_head + 1) & (elts_n - 1);
4418b25d1adSChristian Ehrhardt		/* Prefetch next buffer data. */
4426b3e017eSChristian Ehrhardt		if (pkts_n > 1) {
4436b3e017eSChristian Ehrhardt			volatile void *pkt_addr;
4446b3e017eSChristian Ehrhardt
4456b3e017eSChristian Ehrhardt			pkt_addr = rte_pktmbuf_mtod(*pkts, volatile void *);
4466b3e017eSChristian Ehrhardt			rte_prefetch0(pkt_addr);
4476b3e017eSChristian Ehrhardt		}
4488b25d1adSChristian Ehrhardt		/* Should we enable HW CKSUM offload */
4498b25d1adSChristian Ehrhardt		if (buf->ol_flags &
4508b25d1adSChristian Ehrhardt		    (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM)) {
4516b3e017eSChristian Ehrhardt			wqe->eseg.cs_flags =
4528b25d1adSChristian Ehrhardt				MLX5_ETH_WQE_L3_CSUM |
4538b25d1adSChristian Ehrhardt				MLX5_ETH_WQE_L4_CSUM;
4548b25d1adSChristian Ehrhardt		} else {
4556b3e017eSChristian Ehrhardt			wqe->eseg.cs_flags = 0;
4568b25d1adSChristian Ehrhardt		}
4576b3e017eSChristian Ehrhardt		raw  = (uint8_t *)(uintptr_t)&wqe->eseg.inline_hdr[0];
4586b3e017eSChristian Ehrhardt		/* Start the know and common part of the WQE structure. */
4596b3e017eSChristian Ehrhardt		wqe->ctrl[0] = htonl((txq->wqe_ci << 8) | MLX5_OPCODE_SEND);
4606b3e017eSChristian Ehrhardt		wqe->ctrl[2] = 0;
4616b3e017eSChristian Ehrhardt		wqe->ctrl[3] = 0;
4626b3e017eSChristian Ehrhardt		wqe->eseg.rsvd0 = 0;
4636b3e017eSChristian Ehrhardt		wqe->eseg.rsvd1 = 0;
4646b3e017eSChristian Ehrhardt		wqe->eseg.mss = 0;
4656b3e017eSChristian Ehrhardt		wqe->eseg.rsvd2 = 0;
4666b3e017eSChristian Ehrhardt		/* Start by copying the Ethernet Header. */
4676b3e017eSChristian Ehrhardt		memcpy((uint8_t *)raw, ((uint8_t *)addr), 16);
4686b3e017eSChristian Ehrhardt		length -= MLX5_WQE_DWORD_SIZE;
4696b3e017eSChristian Ehrhardt		addr += MLX5_WQE_DWORD_SIZE;
4706b3e017eSChristian Ehrhardt		/* Replace the Ethernet type by the VLAN if necessary. */
4716b3e017eSChristian Ehrhardt		if (buf->ol_flags & PKT_TX_VLAN_PKT) {
4726b3e017eSChristian Ehrhardt			uint32_t vlan = htonl(0x81000000 | buf->vlan_tci);
4736b3e017eSChristian Ehrhardt
4746b3e017eSChristian Ehrhardt			memcpy((uint8_t *)(raw + MLX5_WQE_DWORD_SIZE -
4756b3e017eSChristian Ehrhardt					   sizeof(vlan)),
4766b3e017eSChristian Ehrhardt			       &vlan, sizeof(vlan));
4776b3e017eSChristian Ehrhardt			addr -= sizeof(vlan);
4786b3e017eSChristian Ehrhardt			length += sizeof(vlan);
4796b3e017eSChristian Ehrhardt		}
4806b3e017eSChristian Ehrhardt		/* Inline if enough room. */
4816b3e017eSChristian Ehrhardt		if (txq->max_inline != 0) {
4826b3e017eSChristian Ehrhardt			uintptr_t end =
4836b3e017eSChristian Ehrhardt				(uintptr_t)&(*txq->wqes)[1 << txq->wqe_n];
4846b3e017eSChristian Ehrhardt			uint16_t max_inline =
4856b3e017eSChristian Ehrhardt				txq->max_inline * RTE_CACHE_LINE_SIZE;
4866b3e017eSChristian Ehrhardt			uint16_t pkt_inline_sz = MLX5_WQE_DWORD_SIZE;
4876b3e017eSChristian Ehrhardt			uint16_t room;
4886b3e017eSChristian Ehrhardt
4896b3e017eSChristian Ehrhardt			raw += MLX5_WQE_DWORD_SIZE;
4906b3e017eSChristian Ehrhardt			room = end - (uintptr_t)raw;
4916b3e017eSChristian Ehrhardt			if (room > max_inline) {
4926b3e017eSChristian Ehrhardt				uintptr_t addr_end = (addr + max_inline) &
4936b3e017eSChristian Ehrhardt					~(RTE_CACHE_LINE_SIZE - 1);
4946b3e017eSChristian Ehrhardt				uint16_t copy_b = ((addr_end - addr) > length) ?
4956b3e017eSChristian Ehrhardt						  length :
4966b3e017eSChristian Ehrhardt						  (addr_end - addr);
4976b3e017eSChristian Ehrhardt
4986b3e017eSChristian Ehrhardt				rte_memcpy((void *)raw, (void *)addr, copy_b);
4996b3e017eSChristian Ehrhardt				addr += copy_b;
5006b3e017eSChristian Ehrhardt				length -= copy_b;
5016b3e017eSChristian Ehrhardt				pkt_inline_sz += copy_b;
5026b3e017eSChristian Ehrhardt				/* Sanity check. */
5036b3e017eSChristian Ehrhardt				assert(addr <= addr_end);
5046b3e017eSChristian Ehrhardt			}
5056b3e017eSChristian Ehrhardt			/* Store the inlined packet size in the WQE. */
5066b3e017eSChristian Ehrhardt			wqe->eseg.inline_hdr_sz = htons(pkt_inline_sz);
5078b25d1adSChristian Ehrhardt			/*
5086b3e017eSChristian Ehrhardt			 * 2 DWORDs consumed by the WQE header + 1 DSEG +
5096b3e017eSChristian Ehrhardt			 * the size of the inline part of the packet.
5108b25d1adSChristian Ehrhardt			 */
5116b3e017eSChristian Ehrhardt			ds = 2 + MLX5_WQE_DS(pkt_inline_sz - 2);
5126b3e017eSChristian Ehrhardt			if (length > 0) {
5136b3e017eSChristian Ehrhardt				dseg = (struct mlx5_wqe_data_seg *)
5146b3e017eSChristian Ehrhardt					((uintptr_t)wqe +
5156b3e017eSChristian Ehrhardt					 (ds * MLX5_WQE_DWORD_SIZE));
5166b3e017eSChristian Ehrhardt				if ((uintptr_t)dseg >= end)
5176b3e017eSChristian Ehrhardt					dseg = (struct mlx5_wqe_data_seg *)
5186b3e017eSChristian Ehrhardt						((uintptr_t)&(*txq->wqes)[0]);
5196b3e017eSChristian Ehrhardt				goto use_dseg;
5206b3e017eSChristian Ehrhardt			} else if (!segs_n) {
5216b3e017eSChristian Ehrhardt				goto next_pkt;
5226b3e017eSChristian Ehrhardt			} else {
5236b3e017eSChristian Ehrhardt				goto next_seg;
5246b3e017eSChristian Ehrhardt			}
5256b3e017eSChristian Ehrhardt		} else {
5266b3e017eSChristian Ehrhardt			/*
5276b3e017eSChristian Ehrhardt			 * No inline has been done in the packet, only the
5286b3e017eSChristian Ehrhardt			 * Ethernet Header as been stored.
5296b3e017eSChristian Ehrhardt			 */
5306b3e017eSChristian Ehrhardt			wqe->eseg.inline_hdr_sz = htons(MLX5_WQE_DWORD_SIZE);
5316b3e017eSChristian Ehrhardt			dseg = (struct mlx5_wqe_data_seg *)
5326b3e017eSChristian Ehrhardt				((uintptr_t)wqe + (3 * MLX5_WQE_DWORD_SIZE));
5336b3e017eSChristian Ehrhardt			ds = 3;
5346b3e017eSChristian Ehrhardtuse_dseg:
5356b3e017eSChristian Ehrhardt			/* Add the remaining packet as a simple ds. */
5366b3e017eSChristian Ehrhardt			*dseg = (struct mlx5_wqe_data_seg) {
5376b3e017eSChristian Ehrhardt				.addr = htonll(addr),
5386b3e017eSChristian Ehrhardt				.byte_count = htonl(length),
5396b3e017eSChristian Ehrhardt				.lkey = txq_mp2mr(txq, txq_mb2mp(buf)),
5406b3e017eSChristian Ehrhardt			};
5418b25d1adSChristian Ehrhardt			++ds;
5426b3e017eSChristian Ehrhardt			if (!segs_n)
5436b3e017eSChristian Ehrhardt				goto next_pkt;
5446b3e017eSChristian Ehrhardt		}
5456b3e017eSChristian Ehrhardtnext_seg:
5466b3e017eSChristian Ehrhardt		assert(buf);
5476b3e017eSChristian Ehrhardt		assert(ds);
5486b3e017eSChristian Ehrhardt		assert(wqe);
5496b3e017eSChristian Ehrhardt		/*
5506b3e017eSChristian Ehrhardt		 * Spill on next WQE when the current one does not have
5516b3e017eSChristian Ehrhardt		 * enough room left. Size of WQE must a be a multiple
5526b3e017eSChristian Ehrhardt		 * of data segment size.
5536b3e017eSChristian Ehrhardt		 */
5546b3e017eSChristian Ehrhardt		assert(!(MLX5_WQE_SIZE % MLX5_WQE_DWORD_SIZE));
5556b3e017eSChristian Ehrhardt		if (!(ds % (MLX5_WQE_SIZE / MLX5_WQE_DWORD_SIZE))) {
5566b3e017eSChristian Ehrhardt			unsigned int n = (txq->wqe_ci + ((ds + 3) / 4)) &
5576b3e017eSChristian Ehrhardt				((1 << txq->wqe_n) - 1);
5586b3e017eSChristian Ehrhardt
5596b3e017eSChristian Ehrhardt			dseg = (struct mlx5_wqe_data_seg *)
5606b3e017eSChristian Ehrhardt				((uintptr_t)&(*txq->wqes)[n]);
5616b3e017eSChristian Ehrhardt			tx_prefetch_wqe(txq, n + 1);
5626b3e017eSChristian Ehrhardt		} else {
5636b3e017eSChristian Ehrhardt			++dseg;
5646b3e017eSChristian Ehrhardt		}
5656b3e017eSChristian Ehrhardt		++ds;
5666b3e017eSChristian Ehrhardt		buf = buf->next;
5676b3e017eSChristian Ehrhardt		assert(buf);
5686b3e017eSChristian Ehrhardt		length = DATA_LEN(buf);
5698b25d1adSChristian Ehrhardt#ifdef MLX5_PMD_SOFT_COUNTERS
5706b3e017eSChristian Ehrhardt		total_length += length;
5718b25d1adSChristian Ehrhardt#endif
5726b3e017eSChristian Ehrhardt		/* Store segment information. */
5736b3e017eSChristian Ehrhardt		*dseg = (struct mlx5_wqe_data_seg) {
5746b3e017eSChristian Ehrhardt			.addr = htonll(rte_pktmbuf_mtod(buf, uintptr_t)),
5756b3e017eSChristian Ehrhardt			.byte_count = htonl(length),
5766b3e017eSChristian Ehrhardt			.lkey = txq_mp2mr(txq, txq_mb2mp(buf)),
5776b3e017eSChristian Ehrhardt		};
5786b3e017eSChristian Ehrhardt		(*txq->elts)[elts_head] = buf;
5796b3e017eSChristian Ehrhardt		elts_head = (elts_head + 1) & (elts_n - 1);
5806b3e017eSChristian Ehrhardt		++j;
5816b3e017eSChristian Ehrhardt		--segs_n;
5826b3e017eSChristian Ehrhardt		if (segs_n)
5836b3e017eSChristian Ehrhardt			goto next_seg;
5846b3e017eSChristian Ehrhardt		else
5856b3e017eSChristian Ehrhardt			--pkts_n;
5866b3e017eSChristian Ehrhardtnext_pkt:
5876b3e017eSChristian Ehrhardt		++i;
5886b3e017eSChristian Ehrhardt		wqe->ctrl[1] = htonl(txq->qp_num_8s | ds);
5896b3e017eSChristian Ehrhardt		txq->wqe_ci += (ds + 3) / 4;
5908b25d1adSChristian Ehrhardt#ifdef MLX5_PMD_SOFT_COUNTERS
5918b25d1adSChristian Ehrhardt		/* Increment sent bytes counter. */
5926b3e017eSChristian Ehrhardt		txq->stats.obytes += total_length;
5938b25d1adSChristian Ehrhardt#endif
5948b25d1adSChristian Ehrhardt	} while (pkts_n);
5958b25d1adSChristian Ehrhardt	/* Take a shortcut if nothing must be sent. */
5968b25d1adSChristian Ehrhardt	if (unlikely(i == 0))
5978b25d1adSChristian Ehrhardt		return 0;
5988b25d1adSChristian Ehrhardt	/* Check whether completion threshold has been reached. */
5998b25d1adSChristian Ehrhardt	comp = txq->elts_comp + i + j;
6008b25d1adSChristian Ehrhardt	if (comp >= MLX5_TX_COMP_THRESH) {
6018b25d1adSChristian Ehrhardt		/* Request completion on last WQE. */
6026b3e017eSChristian Ehrhardt		wqe->ctrl[2] = htonl(8);
6038b25d1adSChristian Ehrhardt		/* Save elts_head in unused "immediate" field of WQE. */
6046b3e017eSChristian Ehrhardt		wqe->ctrl[3] = elts_head;
6058b25d1adSChristian Ehrhardt		txq->elts_comp = 0;
6068b25d1adSChristian Ehrhardt	} else {
6078b25d1adSChristian Ehrhardt		txq->elts_comp = comp;
6088b25d1adSChristian Ehrhardt	}
6098b25d1adSChristian Ehrhardt#ifdef MLX5_PMD_SOFT_COUNTERS
6108b25d1adSChristian Ehrhardt	/* Increment sent packets counter. */
6118b25d1adSChristian Ehrhardt	txq->stats.opackets += i;
6128b25d1adSChristian Ehrhardt#endif
6138b25d1adSChristian Ehrhardt	/* Ring QP doorbell. */
614ce3d555eSChristian Ehrhardt	mlx5_tx_dbrec(txq, (volatile struct mlx5_wqe *)wqe);
6158b25d1adSChristian Ehrhardt	txq->elts_head = elts_head;
6168b25d1adSChristian Ehrhardt	return i;
61797f17497SC.J. Collier}
61897f17497SC.J. Collier
6198b25d1adSChristian Ehrhardt/**
6208b25d1adSChristian Ehrhardt * Open a MPW session.
6218b25d1adSChristian Ehrhardt *
6228b25d1adSChristian Ehrhardt * @param txq
6238b25d1adSChristian Ehrhardt *   Pointer to TX queue structure.
6248b25d1adSChristian Ehrhardt * @param mpw
6258b25d1adSChristian Ehrhardt *   Pointer to MPW session structure.
6268b25d1adSChristian Ehrhardt * @param length
6278b25d1adSChristian Ehrhardt *   Packet length.
6288b25d1adSChristian Ehrhardt */
6298b25d1adSChristian Ehrhardtstatic inline void
6308b25d1adSChristian Ehrhardtmlx5_mpw_new(struct txq *txq, struct mlx5_mpw *mpw, uint32_t length)
6318b25d1adSChristian Ehrhardt{
6326b3e017eSChristian Ehrhardt	uint16_t idx = txq->wqe_ci & ((1 << txq->wqe_n) - 1);
6338b25d1adSChristian Ehrhardt	volatile struct mlx5_wqe_data_seg (*dseg)[MLX5_MPW_DSEG_MAX] =
6348b25d1adSChristian Ehrhardt		(volatile struct mlx5_wqe_data_seg (*)[])
6356b3e017eSChristian Ehrhardt		(uintptr_t)&(*txq->wqes)[(idx + 1) & ((1 << txq->wqe_n) - 1)];
6368b25d1adSChristian Ehrhardt
6378b25d1adSChristian Ehrhardt	mpw->state = MLX5_MPW_STATE_OPENED;
6388b25d1adSChristian Ehrhardt	mpw->pkts_n = 0;
6398b25d1adSChristian Ehrhardt	mpw->len = length;
6408b25d1adSChristian Ehrhardt	mpw->total_len = 0;
6416b3e017eSChristian Ehrhardt	mpw->wqe = (volatile struct mlx5_wqe *)&(*txq->wqes)[idx].hdr;
6426b3e017eSChristian Ehrhardt	mpw->wqe->eseg.mss = htons(length);
6436b3e017eSChristian Ehrhardt	mpw->wqe->eseg.inline_hdr_sz = 0;
6446b3e017eSChristian Ehrhardt	mpw->wqe->eseg.rsvd0 = 0;
6456b3e017eSChristian Ehrhardt	mpw->wqe->eseg.rsvd1 = 0;
6466b3e017eSChristian Ehrhardt	mpw->wqe->eseg.rsvd2 = 0;
6476b3e017eSChristian Ehrhardt	mpw->wqe->ctrl[0] = htonl((MLX5_OPC_MOD_MPW << 24) |
6486b3e017eSChristian Ehrhardt				  (txq->wqe_ci << 8) | MLX5_OPCODE_TSO);
6496b3e017eSChristian Ehrhardt	mpw->wqe->ctrl[2] = 0;
6506b3e017eSChristian Ehrhardt	mpw->wqe->ctrl[3] = 0;
6516b3e017eSChristian Ehrhardt	mpw->data.dseg[0] = (volatile struct mlx5_wqe_data_seg *)
6526b3e017eSChristian Ehrhardt		(((uintptr_t)mpw->wqe) + (2 * MLX5_WQE_DWORD_SIZE));
6536b3e017eSChristian Ehrhardt	mpw->data.dseg[1] = (volatile struct mlx5_wqe_data_seg *)
6546b3e017eSChristian Ehrhardt		(((uintptr_t)mpw->wqe) + (3 * MLX5_WQE_DWORD_SIZE));
6558b25d1adSChristian Ehrhardt	mpw->data.dseg[2] = &(*dseg)[0];
6568b25d1adSChristian Ehrhardt	mpw->data.dseg[3] = &(*dseg)[1];
6578b25d1adSChristian Ehrhardt	mpw->data.dseg[4] = &(*dseg)[2];
6588b25d1adSChristian Ehrhardt}
65997f17497SC.J. Collier
66097f17497SC.J. Collier/**
6618b25d1adSChristian Ehrhardt * Close a MPW session.
6628b25d1adSChristian Ehrhardt *
6638b25d1adSChristian Ehrhardt * @param txq
6648b25d1adSChristian Ehrhardt *   Pointer to TX queue structure.
6658b25d1adSChristian Ehrhardt * @param mpw
6668b25d1adSChristian Ehrhardt *   Pointer to MPW session structure.
6678b25d1adSChristian Ehrhardt */
6688b25d1adSChristian Ehrhardtstatic inline void
6698b25d1adSChristian Ehrhardtmlx5_mpw_close(struct txq *txq, struct mlx5_mpw *mpw)
6708b25d1adSChristian Ehrhardt{
6718b25d1adSChristian Ehrhardt	unsigned int num = mpw->pkts_n;
6728b25d1adSChristian Ehrhardt
6738b25d1adSChristian Ehrhardt	/*
6748b25d1adSChristian Ehrhardt	 * Store size in multiple of 16 bytes. Control and Ethernet segments
6758b25d1adSChristian Ehrhardt	 * count as 2.
6768b25d1adSChristian Ehrhardt	 */
6776b3e017eSChristian Ehrhardt	mpw->wqe->ctrl[1] = htonl(txq->qp_num_8s | (2 + num));
6788b25d1adSChristian Ehrhardt	mpw->state = MLX5_MPW_STATE_CLOSED;
6798b25d1adSChristian Ehrhardt	if (num < 3)
6808b25d1adSChristian Ehrhardt		++txq->wqe_ci;
6818b25d1adSChristian Ehrhardt	else
6828b25d1adSChristian Ehrhardt		txq->wqe_ci += 2;
6838b25d1adSChristian Ehrhardt	tx_prefetch_wqe(txq, txq->wqe_ci);
6848b25d1adSChristian Ehrhardt	tx_prefetch_wqe(txq, txq->wqe_ci + 1);
6858b25d1adSChristian Ehrhardt}
6868b25d1adSChristian Ehrhardt
6878b25d1adSChristian Ehrhardt/**
6888b25d1adSChristian Ehrhardt * DPDK callback for TX with MPW support.
68997f17497SC.J. Collier *
69097f17497SC.J. Collier * @param dpdk_txq
69197f17497SC.J. Collier *   Generic pointer to TX queue structure.
69297f17497SC.J. Collier * @param[in] pkts
69397f17497SC.J. Collier *   Packets to transmit.
69497f17497SC.J. Collier * @param pkts_n
69597f17497SC.J. Collier *   Number of packets in array.
69697f17497SC.J. Collier *
69797f17497SC.J. Collier * @return
69897f17497SC.J. Collier *   Number of packets successfully transmitted (<= pkts_n).
69997f17497SC.J. Collier */
70097f17497SC.J. Collieruint16_t
7018b25d1adSChristian Ehrhardtmlx5_tx_burst_mpw(void *dpdk_txq, struct rte_mbuf **pkts, uint16_t pkts_n)
70297f17497SC.J. Collier{
70397f17497SC.J. Collier	struct txq *txq = (struct txq *)dpdk_txq;
7048b25d1adSChristian Ehrhardt	uint16_t elts_head = txq->elts_head;
7056b3e017eSChristian Ehrhardt	const unsigned int elts_n = 1 << txq->elts_n;
7068b25d1adSChristian Ehrhardt	unsigned int i = 0;
7078b25d1adSChristian Ehrhardt	unsigned int j = 0;
70897f17497SC.J. Collier	unsigned int max;
7098b25d1adSChristian Ehrhardt	unsigned int comp;
7108b25d1adSChristian Ehrhardt	struct mlx5_mpw mpw = {
7118b25d1adSChristian Ehrhardt		.state = MLX5_MPW_STATE_CLOSED,
7128b25d1adSChristian Ehrhardt	};
71397f17497SC.J. Collier
7148b25d1adSChristian Ehrhardt	if (unlikely(!pkts_n))
7158b25d1adSChristian Ehrhardt		return 0;
71697f17497SC.J. Collier	/* Prefetch first packet cacheline. */
7178b25d1adSChristian Ehrhardt	tx_prefetch_cqe(txq, txq->cq_ci);
7188b25d1adSChristian Ehrhardt	tx_prefetch_wqe(txq, txq->wqe_ci);
7198b25d1adSChristian Ehrhardt	tx_prefetch_wqe(txq, txq->wqe_ci + 1);
7208b25d1adSChristian Ehrhardt	/* Start processing. */
72197f17497SC.J. Collier	txq_complete(txq);
72297f17497SC.J. Collier	max = (elts_n - (elts_head - txq->elts_tail));
72397f17497SC.J. Collier	if (max > elts_n)
72497f17497SC.J. Collier		max -= elts_n;
7258b25d1adSChristian Ehrhardt	do {
7268b25d1adSChristian Ehrhardt		struct rte_mbuf *buf = *(pkts++);
7278b25d1adSChristian Ehrhardt		unsigned int elts_head_next;
7288b25d1adSChristian Ehrhardt		uint32_t length;
7298b25d1adSChristian Ehrhardt		unsigned int segs_n = buf->nb_segs;
7308b25d1adSChristian Ehrhardt		uint32_t cs_flags = 0;
7318b25d1adSChristian Ehrhardt
7328b25d1adSChristian Ehrhardt		/*
7338b25d1adSChristian Ehrhardt		 * Make sure there is enough room to store this packet and
7348b25d1adSChristian Ehrhardt		 * that one ring entry remains unused.
7358b25d1adSChristian Ehrhardt		 */
7368b25d1adSChristian Ehrhardt		assert(segs_n);
7378b25d1adSChristian Ehrhardt		if (max < segs_n + 1)
7388b25d1adSChristian Ehrhardt			break;
7398b25d1adSChristian Ehrhardt		/* Do not bother with large packets MPW cannot handle. */
74047d9763aSLuca Boccassi		if (segs_n > MLX5_MPW_DSEG_MAX) {
74147d9763aSLuca Boccassi			txq->stats.oerrors++;
7428b25d1adSChristian Ehrhardt			break;
74347d9763aSLuca Boccassi		}
7448b25d1adSChristian Ehrhardt		max -= segs_n;
7458b25d1adSChristian Ehrhardt		--pkts_n;
7468b25d1adSChristian Ehrhardt		/* Should we enable HW CKSUM offload */
7478b25d1adSChristian Ehrhardt		if (buf->ol_flags &
7488b25d1adSChristian Ehrhardt		    (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM))
7498b25d1adSChristian Ehrhardt			cs_flags = MLX5_ETH_WQE_L3_CSUM | MLX5_ETH_WQE_L4_CSUM;
7508b25d1adSChristian Ehrhardt		/* Retrieve packet information. */
7518b25d1adSChristian Ehrhardt		length = PKT_LEN(buf);
7528b25d1adSChristian Ehrhardt		assert(length);
7538b25d1adSChristian Ehrhardt		/* Start new session if packet differs. */
7548b25d1adSChristian Ehrhardt		if ((mpw.state == MLX5_MPW_STATE_OPENED) &&
7558b25d1adSChristian Ehrhardt		    ((mpw.len != length) ||
7568b25d1adSChristian Ehrhardt		     (segs_n != 1) ||
7576b3e017eSChristian Ehrhardt		     (mpw.wqe->eseg.cs_flags != cs_flags)))
7588b25d1adSChristian Ehrhardt			mlx5_mpw_close(txq, &mpw);
7598b25d1adSChristian Ehrhardt		if (mpw.state == MLX5_MPW_STATE_CLOSED) {
7608b25d1adSChristian Ehrhardt			mlx5_mpw_new(txq, &mpw, length);
7616b3e017eSChristian Ehrhardt			mpw.wqe->eseg.cs_flags = cs_flags;
7628b25d1adSChristian Ehrhardt		}
7638b25d1adSChristian Ehrhardt		/* Multi-segment packets must be alone in their MPW. */
7648b25d1adSChristian Ehrhardt		assert((segs_n == 1) || (mpw.pkts_n == 0));
7658b25d1adSChristian Ehrhardt#if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
7668b25d1adSChristian Ehrhardt		length = 0;
7678b25d1adSChristian Ehrhardt#endif
7688b25d1adSChristian Ehrhardt		do {
7698b25d1adSChristian Ehrhardt			volatile struct mlx5_wqe_data_seg *dseg;
7708b25d1adSChristian Ehrhardt			uintptr_t addr;
7718b25d1adSChristian Ehrhardt
7728b25d1adSChristian Ehrhardt			elts_head_next = (elts_head + 1) & (elts_n - 1);
7738b25d1adSChristian Ehrhardt			assert(buf);
7748b25d1adSChristian Ehrhardt			(*txq->elts)[elts_head] = buf;
7758b25d1adSChristian Ehrhardt			dseg = mpw.data.dseg[mpw.pkts_n];
7768b25d1adSChristian Ehrhardt			addr = rte_pktmbuf_mtod(buf, uintptr_t);
7778b25d1adSChristian Ehrhardt			*dseg = (struct mlx5_wqe_data_seg){
7788b25d1adSChristian Ehrhardt				.byte_count = htonl(DATA_LEN(buf)),
7798b25d1adSChristian Ehrhardt				.lkey = txq_mp2mr(txq, txq_mb2mp(buf)),
7808b25d1adSChristian Ehrhardt				.addr = htonll(addr),
7818b25d1adSChristian Ehrhardt			};
7828b25d1adSChristian Ehrhardt			elts_head = elts_head_next;
7838b25d1adSChristian Ehrhardt#if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
7848b25d1adSChristian Ehrhardt			length += DATA_LEN(buf);
7858b25d1adSChristian Ehrhardt#endif
7868b25d1adSChristian Ehrhardt			buf = buf->next;
7878b25d1adSChristian Ehrhardt			++mpw.pkts_n;
7888b25d1adSChristian Ehrhardt			++j;
7898b25d1adSChristian Ehrhardt		} while (--segs_n);
7908b25d1adSChristian Ehrhardt		assert(length == mpw.len);
7918b25d1adSChristian Ehrhardt		if (mpw.pkts_n == MLX5_MPW_DSEG_MAX)
7928b25d1adSChristian Ehrhardt			mlx5_mpw_close(txq, &mpw);
7938b25d1adSChristian Ehrhardt		elts_head = elts_head_next;
7948b25d1adSChristian Ehrhardt#ifdef MLX5_PMD_SOFT_COUNTERS
7958b25d1adSChristian Ehrhardt		/* Increment sent bytes counter. */
7968b25d1adSChristian Ehrhardt		txq->stats.obytes += length;
7978b25d1adSChristian Ehrhardt#endif
7988b25d1adSChristian Ehrhardt		++i;
7998b25d1adSChristian Ehrhardt	} while (pkts_n);
8008b25d1adSChristian Ehrhardt	/* Take a shortcut if nothing must be sent. */
8018b25d1adSChristian Ehrhardt	if (unlikely(i == 0))
80297f17497SC.J. Collier		return 0;
8038b25d1adSChristian Ehrhardt	/* Check whether completion threshold has been reached. */
8048b25d1adSChristian Ehrhardt	/* "j" includes both packets and segments. */
8058b25d1adSChristian Ehrhardt	comp = txq->elts_comp + j;
8068b25d1adSChristian Ehrhardt	if (comp >= MLX5_TX_COMP_THRESH) {
8076b3e017eSChristian Ehrhardt		volatile struct mlx5_wqe *wqe = mpw.wqe;
8088b25d1adSChristian Ehrhardt
8098b25d1adSChristian Ehrhardt		/* Request completion on last WQE. */
8106b3e017eSChristian Ehrhardt		wqe->ctrl[2] = htonl(8);
8118b25d1adSChristian Ehrhardt		/* Save elts_head in unused "immediate" field of WQE. */
8126b3e017eSChristian Ehrhardt		wqe->ctrl[3] = elts_head;
8138b25d1adSChristian Ehrhardt		txq->elts_comp = 0;
8148b25d1adSChristian Ehrhardt	} else {
8158b25d1adSChristian Ehrhardt		txq->elts_comp = comp;
8168b25d1adSChristian Ehrhardt	}
81797f17497SC.J. Collier#ifdef MLX5_PMD_SOFT_COUNTERS
8188b25d1adSChristian Ehrhardt	/* Increment sent packets counter. */
8198b25d1adSChristian Ehrhardt	txq->stats.opackets += i;
82097f17497SC.J. Collier#endif
8218b25d1adSChristian Ehrhardt	/* Ring QP doorbell. */
8228b25d1adSChristian Ehrhardt	if (mpw.state == MLX5_MPW_STATE_OPENED)
8238b25d1adSChristian Ehrhardt		mlx5_mpw_close(txq, &mpw);
824ce3d555eSChristian Ehrhardt	mlx5_tx_dbrec(txq, mpw.wqe);
8258b25d1adSChristian Ehrhardt	txq->elts_head = elts_head;
8268b25d1adSChristian Ehrhardt	return i;
8278b25d1adSChristian Ehrhardt}
8288b25d1adSChristian Ehrhardt
8298b25d1adSChristian Ehrhardt/**
8308b25d1adSChristian Ehrhardt * Open a MPW inline session.
8318b25d1adSChristian Ehrhardt *
8328b25d1adSChristian Ehrhardt * @param txq
8338b25d1adSChristian Ehrhardt *   Pointer to TX queue structure.
8348b25d1adSChristian Ehrhardt * @param mpw
8358b25d1adSChristian Ehrhardt *   Pointer to MPW session structure.
8368b25d1adSChristian Ehrhardt * @param length
8378b25d1adSChristian Ehrhardt *   Packet length.
8388b25d1adSChristian Ehrhardt */
8398b25d1adSChristian Ehrhardtstatic inline void
8408b25d1adSChristian Ehrhardtmlx5_mpw_inline_new(struct txq *txq, struct mlx5_mpw *mpw, uint32_t length)
8418b25d1adSChristian Ehrhardt{
8426b3e017eSChristian Ehrhardt	uint16_t idx = txq->wqe_ci & ((1 << txq->wqe_n) - 1);
8436b3e017eSChristian Ehrhardt	struct mlx5_wqe_inl_small *inl;
8448b25d1adSChristian Ehrhardt
8458b25d1adSChristian Ehrhardt	mpw->state = MLX5_MPW_INL_STATE_OPENED;
8468b25d1adSChristian Ehrhardt	mpw->pkts_n = 0;
8478b25d1adSChristian Ehrhardt	mpw->len = length;
8488b25d1adSChristian Ehrhardt	mpw->total_len = 0;
8496b3e017eSChristian Ehrhardt	mpw->wqe = (volatile struct mlx5_wqe *)&(*txq->wqes)[idx].hdr;
8506b3e017eSChristian Ehrhardt	mpw->wqe->ctrl[0] = htonl((MLX5_OPC_MOD_MPW << 24) |
8516b3e017eSChristian Ehrhardt				  (txq->wqe_ci << 8) |
8526b3e017eSChristian Ehrhardt				  MLX5_OPCODE_TSO);
8536b3e017eSChristian Ehrhardt	mpw->wqe->ctrl[2] = 0;
8546b3e017eSChristian Ehrhardt	mpw->wqe->ctrl[3] = 0;
8556b3e017eSChristian Ehrhardt	mpw->wqe->eseg.mss = htons(length);
8566b3e017eSChristian Ehrhardt	mpw->wqe->eseg.inline_hdr_sz = 0;
8576b3e017eSChristian Ehrhardt	mpw->wqe->eseg.cs_flags = 0;
8586b3e017eSChristian Ehrhardt	mpw->wqe->eseg.rsvd0 = 0;
8596b3e017eSChristian Ehrhardt	mpw->wqe->eseg.rsvd1 = 0;
8606b3e017eSChristian Ehrhardt	mpw->wqe->eseg.rsvd2 = 0;
8616b3e017eSChristian Ehrhardt	inl = (struct mlx5_wqe_inl_small *)
8626b3e017eSChristian Ehrhardt		(((uintptr_t)mpw->wqe) + 2 * MLX5_WQE_DWORD_SIZE);
8636b3e017eSChristian Ehrhardt	mpw->data.raw = (uint8_t *)&inl->raw;
8648b25d1adSChristian Ehrhardt}
8658b25d1adSChristian Ehrhardt
8668b25d1adSChristian Ehrhardt/**
8678b25d1adSChristian Ehrhardt * Close a MPW inline session.
8688b25d1adSChristian Ehrhardt *
8698b25d1adSChristian Ehrhardt * @param txq
8708b25d1adSChristian Ehrhardt *   Pointer to TX queue structure.
8718b25d1adSChristian Ehrhardt * @param mpw
8728b25d1adSChristian Ehrhardt *   Pointer to MPW session structure.
8738b25d1adSChristian Ehrhardt */
8748b25d1adSChristian Ehrhardtstatic inline void
8758b25d1adSChristian Ehrhardtmlx5_mpw_inline_close(struct txq *txq, struct mlx5_mpw *mpw)
8768b25d1adSChristian Ehrhardt{
8778b25d1adSChristian Ehrhardt	unsigned int size;
8786b3e017eSChristian Ehrhardt	struct mlx5_wqe_inl_small *inl = (struct mlx5_wqe_inl_small *)
8796b3e017eSChristian Ehrhardt		(((uintptr_t)mpw->wqe) + (2 * MLX5_WQE_DWORD_SIZE));
8808b25d1adSChristian Ehrhardt
8816b3e017eSChristian Ehrhardt	size = MLX5_WQE_SIZE - MLX5_MWQE64_INL_DATA + mpw->total_len;
8828b25d1adSChristian Ehrhardt	/*
8838b25d1adSChristian Ehrhardt	 * Store size in multiple of 16 bytes. Control and Ethernet segments
8848b25d1adSChristian Ehrhardt	 * count as 2.
8858b25d1adSChristian Ehrhardt	 */
8866b3e017eSChristian Ehrhardt	mpw->wqe->ctrl[1] = htonl(txq->qp_num_8s | MLX5_WQE_DS(size));
8878b25d1adSChristian Ehrhardt	mpw->state = MLX5_MPW_STATE_CLOSED;
8886b3e017eSChristian Ehrhardt	inl->byte_cnt = htonl(mpw->total_len | MLX5_INLINE_SEG);
8896b3e017eSChristian Ehrhardt	txq->wqe_ci += (size + (MLX5_WQE_SIZE - 1)) / MLX5_WQE_SIZE;
8908b25d1adSChristian Ehrhardt}
8918b25d1adSChristian Ehrhardt
8928b25d1adSChristian Ehrhardt/**
8938b25d1adSChristian Ehrhardt * DPDK callback for TX with MPW inline support.
8948b25d1adSChristian Ehrhardt *
8958b25d1adSChristian Ehrhardt * @param dpdk_txq
8968b25d1adSChristian Ehrhardt *   Generic pointer to TX queue structure.
8978b25d1adSChristian Ehrhardt * @param[in] pkts
8988b25d1adSChristian Ehrhardt *   Packets to transmit.
8998b25d1adSChristian Ehrhardt * @param pkts_n
9008b25d1adSChristian Ehrhardt *   Number of packets in array.
9018b25d1adSChristian Ehrhardt *
9028b25d1adSChristian Ehrhardt * @return
9038b25d1adSChristian Ehrhardt *   Number of packets successfully transmitted (<= pkts_n).
9048b25d1adSChristian Ehrhardt */
9058b25d1adSChristian Ehrhardtuint16_t
9068b25d1adSChristian Ehrhardtmlx5_tx_burst_mpw_inline(void *dpdk_txq, struct rte_mbuf **pkts,
9078b25d1adSChristian Ehrhardt			 uint16_t pkts_n)
9088b25d1adSChristian Ehrhardt{
9098b25d1adSChristian Ehrhardt	struct txq *txq = (struct txq *)dpdk_txq;
9108b25d1adSChristian Ehrhardt	uint16_t elts_head = txq->elts_head;
9116b3e017eSChristian Ehrhardt	const unsigned int elts_n = 1 << txq->elts_n;
9128b25d1adSChristian Ehrhardt	unsigned int i = 0;
9138b25d1adSChristian Ehrhardt	unsigned int j = 0;
9148b25d1adSChristian Ehrhardt	unsigned int max;
9158b25d1adSChristian Ehrhardt	unsigned int comp;
91632e04ea0SChristian Ehrhardt	unsigned int inline_room = txq->max_inline * RTE_CACHE_LINE_SIZE;
9178b25d1adSChristian Ehrhardt	struct mlx5_mpw mpw = {
9188b25d1adSChristian Ehrhardt		.state = MLX5_MPW_STATE_CLOSED,
9198b25d1adSChristian Ehrhardt	};
9208b25d1adSChristian Ehrhardt
9218b25d1adSChristian Ehrhardt	if (unlikely(!pkts_n))
9228b25d1adSChristian Ehrhardt		return 0;
9238b25d1adSChristian Ehrhardt	/* Prefetch first packet cacheline. */
9248b25d1adSChristian Ehrhardt	tx_prefetch_cqe(txq, txq->cq_ci);
9258b25d1adSChristian Ehrhardt	tx_prefetch_wqe(txq, txq->wqe_ci);
9268b25d1adSChristian Ehrhardt	tx_prefetch_wqe(txq, txq->wqe_ci + 1);
9278b25d1adSChristian Ehrhardt	/* Start processing. */
9288b25d1adSChristian Ehrhardt	txq_complete(txq);
9298b25d1adSChristian Ehrhardt	max = (elts_n - (elts_head - txq->elts_tail));
9308b25d1adSChristian Ehrhardt	if (max > elts_n)
9318b25d1adSChristian Ehrhardt		max -= elts_n;
9328b25d1adSChristian Ehrhardt	do {
9338b25d1adSChristian Ehrhardt		struct rte_mbuf *buf = *(pkts++);
9348b25d1adSChristian Ehrhardt		unsigned int elts_head_next;
9358b25d1adSChristian Ehrhardt		uintptr_t addr;
9368b25d1adSChristian Ehrhardt		uint32_t length;
9378b25d1adSChristian Ehrhardt		unsigned int segs_n = buf->nb_segs;
9388b25d1adSChristian Ehrhardt		uint32_t cs_flags = 0;
9398b25d1adSChristian Ehrhardt
9408b25d1adSChristian Ehrhardt		/*
9418b25d1adSChristian Ehrhardt		 * Make sure there is enough room to store this packet and
9428b25d1adSChristian Ehrhardt		 * that one ring entry remains unused.
9438b25d1adSChristian Ehrhardt		 */
9448b25d1adSChristian Ehrhardt		assert(segs_n);
9458b25d1adSChristian Ehrhardt		if (max < segs_n + 1)
9468b25d1adSChristian Ehrhardt			break;
9478b25d1adSChristian Ehrhardt		/* Do not bother with large packets MPW cannot handle. */
94847d9763aSLuca Boccassi		if (segs_n > MLX5_MPW_DSEG_MAX) {
94947d9763aSLuca Boccassi			txq->stats.oerrors++;
9508b25d1adSChristian Ehrhardt			break;
95147d9763aSLuca Boccassi		}
9528b25d1adSChristian Ehrhardt		max -= segs_n;
9538b25d1adSChristian Ehrhardt		--pkts_n;
95497f17497SC.J. Collier		/* Should we enable HW CKSUM offload */
95597f17497SC.J. Collier		if (buf->ol_flags &
9568b25d1adSChristian Ehrhardt		    (PKT_TX_IP_CKSUM | PKT_TX_TCP_CKSUM | PKT_TX_UDP_CKSUM))
9578b25d1adSChristian Ehrhardt			cs_flags = MLX5_ETH_WQE_L3_CSUM | MLX5_ETH_WQE_L4_CSUM;
9588b25d1adSChristian Ehrhardt		/* Retrieve packet information. */
9598b25d1adSChristian Ehrhardt		length = PKT_LEN(buf);
9608b25d1adSChristian Ehrhardt		/* Start new session if packet differs. */
9618b25d1adSChristian Ehrhardt		if (mpw.state == MLX5_MPW_STATE_OPENED) {
9628b25d1adSChristian Ehrhardt			if ((mpw.len != length) ||
9638b25d1adSChristian Ehrhardt			    (segs_n != 1) ||
9646b3e017eSChristian Ehrhardt			    (mpw.wqe->eseg.cs_flags != cs_flags))
9658b25d1adSChristian Ehrhardt				mlx5_mpw_close(txq, &mpw);
9668b25d1adSChristian Ehrhardt		} else if (mpw.state == MLX5_MPW_INL_STATE_OPENED) {
9678b25d1adSChristian Ehrhardt			if ((mpw.len != length) ||
9688b25d1adSChristian Ehrhardt			    (segs_n != 1) ||
9698b25d1adSChristian Ehrhardt			    (length > inline_room) ||
9706b3e017eSChristian Ehrhardt			    (mpw.wqe->eseg.cs_flags != cs_flags)) {
9718b25d1adSChristian Ehrhardt				mlx5_mpw_inline_close(txq, &mpw);
97232e04ea0SChristian Ehrhardt				inline_room =
97332e04ea0SChristian Ehrhardt					txq->max_inline * RTE_CACHE_LINE_SIZE;
9748b25d1adSChristian Ehrhardt			}
97597f17497SC.J. Collier		}
9768b25d1adSChristian Ehrhardt		if (mpw.state == MLX5_MPW_STATE_CLOSED) {
9778b25d1adSChristian Ehrhardt			if ((segs_n != 1) ||
9788b25d1adSChristian Ehrhardt			    (length > inline_room)) {
9798b25d1adSChristian Ehrhardt				mlx5_mpw_new(txq, &mpw, length);
9806b3e017eSChristian Ehrhardt				mpw.wqe->eseg.cs_flags = cs_flags;
9818b25d1adSChristian Ehrhardt			} else {
9828b25d1adSChristian Ehrhardt				mlx5_mpw_inline_new(txq, &mpw, length);
9836b3e017eSChristian Ehrhardt				mpw.wqe->eseg.cs_flags = cs_flags;
98497f17497SC.J. Collier			}
98597f17497SC.J. Collier		}
9868b25d1adSChristian Ehrhardt		/* Multi-segment packets must be alone in their MPW. */
9878b25d1adSChristian Ehrhardt		assert((segs_n == 1) || (mpw.pkts_n == 0));
9888b25d1adSChristian Ehrhardt		if (mpw.state == MLX5_MPW_STATE_OPENED) {
98932e04ea0SChristian Ehrhardt			assert(inline_room ==
99032e04ea0SChristian Ehrhardt			       txq->max_inline * RTE_CACHE_LINE_SIZE);
9918b25d1adSChristian Ehrhardt#if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
9928b25d1adSChristian Ehrhardt			length = 0;
9938b25d1adSChristian Ehrhardt#endif
9948b25d1adSChristian Ehrhardt			do {
9958b25d1adSChristian Ehrhardt				volatile struct mlx5_wqe_data_seg *dseg;
9968b25d1adSChristian Ehrhardt
9978b25d1adSChristian Ehrhardt				elts_head_next =
9988b25d1adSChristian Ehrhardt					(elts_head + 1) & (elts_n - 1);
9998b25d1adSChristian Ehrhardt				assert(buf);
10008b25d1adSChristian Ehrhardt				(*txq->elts)[elts_head] = buf;
10018b25d1adSChristian Ehrhardt				dseg = mpw.data.dseg[mpw.pkts_n];
10028b25d1adSChristian Ehrhardt				addr = rte_pktmbuf_mtod(buf, uintptr_t);
10038b25d1adSChristian Ehrhardt				*dseg = (struct mlx5_wqe_data_seg){
10048b25d1adSChristian Ehrhardt					.byte_count = htonl(DATA_LEN(buf)),
10058b25d1adSChristian Ehrhardt					.lkey = txq_mp2mr(txq, txq_mb2mp(buf)),
10068b25d1adSChristian Ehrhardt					.addr = htonll(addr),
10078b25d1adSChristian Ehrhardt				};
10088b25d1adSChristian Ehrhardt				elts_head = elts_head_next;
10098b25d1adSChristian Ehrhardt#if defined(MLX5_PMD_SOFT_COUNTERS) || !defined(NDEBUG)
10108b25d1adSChristian Ehrhardt				length += DATA_LEN(buf);
10118b25d1adSChristian Ehrhardt#endif
10128b25d1adSChristian Ehrhardt				buf = buf->next;
10138b25d1adSChristian Ehrhardt				++mpw.pkts_n;
10148b25d1adSChristian Ehrhardt				++j;
10158b25d1adSChristian Ehrhardt			} while (--segs_n);
10168b25d1adSChristian Ehrhardt			assert(length == mpw.len);
10178b25d1adSChristian Ehrhardt			if (mpw.pkts_n == MLX5_MPW_DSEG_MAX)
10188b25d1adSChristian Ehrhardt				mlx5_mpw_close(txq, &mpw);