1809f0800SChristian Ehrhardt/*-
2809f0800SChristian Ehrhardt *   BSD LICENSE
3809f0800SChristian Ehrhardt *
4809f0800SChristian Ehrhardt *   Copyright 2016 6WIND S.A.
5809f0800SChristian Ehrhardt *   Copyright 2016 Mellanox.
6809f0800SChristian Ehrhardt *
7809f0800SChristian Ehrhardt *   Redistribution and use in source and binary forms, with or without
8809f0800SChristian Ehrhardt *   modification, are permitted provided that the following conditions
9809f0800SChristian Ehrhardt *   are met:
10809f0800SChristian Ehrhardt *
11809f0800SChristian Ehrhardt *     * Redistributions of source code must retain the above copyright
12809f0800SChristian Ehrhardt *       notice, this list of conditions and the following disclaimer.
13809f0800SChristian Ehrhardt *     * Redistributions in binary form must reproduce the above copyright
14809f0800SChristian Ehrhardt *       notice, this list of conditions and the following disclaimer in
15809f0800SChristian Ehrhardt *       the documentation and/or other materials provided with the
16809f0800SChristian Ehrhardt *       distribution.
17809f0800SChristian Ehrhardt *     * Neither the name of 6WIND S.A. nor the names of its
18809f0800SChristian Ehrhardt *       contributors may be used to endorse or promote products derived
19809f0800SChristian Ehrhardt *       from this software without specific prior written permission.
20809f0800SChristian Ehrhardt *
21809f0800SChristian Ehrhardt *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22809f0800SChristian Ehrhardt *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23809f0800SChristian Ehrhardt *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24809f0800SChristian Ehrhardt *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25809f0800SChristian Ehrhardt *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26809f0800SChristian Ehrhardt *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27809f0800SChristian Ehrhardt *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28809f0800SChristian Ehrhardt *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29809f0800SChristian Ehrhardt *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30809f0800SChristian Ehrhardt *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31809f0800SChristian Ehrhardt *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32809f0800SChristian Ehrhardt */
33809f0800SChristian Ehrhardt
34809f0800SChristian Ehrhardt/* Verbs header. */
35809f0800SChristian Ehrhardt/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
36809f0800SChristian Ehrhardt#ifdef PEDANTIC
37c300e355SChristian Ehrhardt#pragma GCC diagnostic ignored "-Wpedantic"
38809f0800SChristian Ehrhardt#endif
39809f0800SChristian Ehrhardt#include <infiniband/verbs.h>
40809f0800SChristian Ehrhardt#ifdef PEDANTIC
41c300e355SChristian Ehrhardt#pragma GCC diagnostic error "-Wpedantic"
42809f0800SChristian Ehrhardt#endif
43809f0800SChristian Ehrhardt
44809f0800SChristian Ehrhardt/* DPDK headers don't like -pedantic. */
45809f0800SChristian Ehrhardt#ifdef PEDANTIC
46c300e355SChristian Ehrhardt#pragma GCC diagnostic ignored "-Wpedantic"
47809f0800SChristian Ehrhardt#endif
48809f0800SChristian Ehrhardt#include <rte_mempool.h>
49809f0800SChristian Ehrhardt#ifdef PEDANTIC
50c300e355SChristian Ehrhardt#pragma GCC diagnostic error "-Wpedantic"
51809f0800SChristian Ehrhardt#endif
52809f0800SChristian Ehrhardt
53809f0800SChristian Ehrhardt#include "mlx5.h"
54809f0800SChristian Ehrhardt#include "mlx5_rxtx.h"
55809f0800SChristian Ehrhardt
56809f0800SChristian Ehrhardtstruct mlx5_check_mempool_data {
57809f0800SChristian Ehrhardt	int ret;
58809f0800SChristian Ehrhardt	char *start;
59809f0800SChristian Ehrhardt	char *end;
60809f0800SChristian Ehrhardt};
61809f0800SChristian Ehrhardt
62809f0800SChristian Ehrhardt/* Called by mlx5_check_mempool() when iterating the memory chunks. */
63809f0800SChristian Ehrhardtstatic void
64809f0800SChristian Ehrhardtmlx5_check_mempool_cb(struct rte_mempool *mp,
65809f0800SChristian Ehrhardt		      void *opaque, struct rte_mempool_memhdr *memhdr,
66809f0800SChristian Ehrhardt		      unsigned int mem_idx)
67809f0800SChristian Ehrhardt{
68809f0800SChristian Ehrhardt	struct mlx5_check_mempool_data *data = opaque;
69809f0800SChristian Ehrhardt
70809f0800SChristian Ehrhardt	(void)mp;
71809f0800SChristian Ehrhardt	(void)mem_idx;
72809f0800SChristian Ehrhardt
73809f0800SChristian Ehrhardt	/* It already failed, skip the next chunks. */
74809f0800SChristian Ehrhardt	if (data->ret != 0)
75809f0800SChristian Ehrhardt		return;
76809f0800SChristian Ehrhardt	/* It is the first chunk. */
77809f0800SChristian Ehrhardt	if (data->start == NULL && data->end == NULL) {
78809f0800SChristian Ehrhardt		data->start = memhdr->addr;
79809f0800SChristian Ehrhardt		data->end = data->start + memhdr->len;
80809f0800SChristian Ehrhardt		return;
81809f0800SChristian Ehrhardt	}
82809f0800SChristian Ehrhardt	if (data->end == memhdr->addr) {
83809f0800SChristian Ehrhardt		data->end += memhdr->len;
84809f0800SChristian Ehrhardt		return;
85809f0800SChristian Ehrhardt	}
86809f0800SChristian Ehrhardt	if (data->start == (char *)memhdr->addr + memhdr->len) {
87809f0800SChristian Ehrhardt		data->start -= memhdr->len;
88809f0800SChristian Ehrhardt		return;
89809f0800SChristian Ehrhardt	}
90809f0800SChristian Ehrhardt	/* Error, mempool is not virtually contiguous. */
91809f0800SChristian Ehrhardt	data->ret = -1;
92809f0800SChristian Ehrhardt}
93809f0800SChristian Ehrhardt
94809f0800SChristian Ehrhardt/**
95809f0800SChristian Ehrhardt * Check if a mempool can be used: it must be virtually contiguous.
96809f0800SChristian Ehrhardt *
97809f0800SChristian Ehrhardt * @param[in] mp
98809f0800SChristian Ehrhardt *   Pointer to memory pool.
99809f0800SChristian Ehrhardt * @param[out] start
100809f0800SChristian Ehrhardt *   Pointer to the start address of the mempool virtual memory area
101809f0800SChristian Ehrhardt * @param[out] end
102809f0800SChristian Ehrhardt *   Pointer to the end address of the mempool virtual memory area
103809f0800SChristian Ehrhardt *
104809f0800SChristian Ehrhardt * @return
105809f0800SChristian Ehrhardt *   0 on success (mempool is virtually contiguous), -1 on error.
106809f0800SChristian Ehrhardt */
107809f0800SChristian Ehrhardtstatic int mlx5_check_mempool(struct rte_mempool *mp, uintptr_t *start,
108809f0800SChristian Ehrhardt	uintptr_t *end)
109809f0800SChristian Ehrhardt{
110809f0800SChristian Ehrhardt	struct mlx5_check_mempool_data data;
111809f0800SChristian Ehrhardt
112809f0800SChristian Ehrhardt	memset(&data, 0, sizeof(data));
113809f0800SChristian Ehrhardt	rte_mempool_mem_iter(mp, mlx5_check_mempool_cb, &data);
114809f0800SChristian Ehrhardt	*start = (uintptr_t)data.start;
115809f0800SChristian Ehrhardt	*end = (uintptr_t)data.end;
116809f0800SChristian Ehrhardt
117809f0800SChristian Ehrhardt	return data.ret;
118809f0800SChristian Ehrhardt}
119809f0800SChristian Ehrhardt
120809f0800SChristian Ehrhardt/**
121809f0800SChristian Ehrhardt * Register mempool as a memory region.
122809f0800SChristian Ehrhardt *
123809f0800SChristian Ehrhardt * @param pd
124809f0800SChristian Ehrhardt *   Pointer to protection domain.
125809f0800SChristian Ehrhardt * @param mp
126809f0800SChristian Ehrhardt *   Pointer to memory pool.
127809f0800SChristian Ehrhardt *
128809f0800SChristian Ehrhardt * @return
129809f0800SChristian Ehrhardt *   Memory region pointer, NULL in case of error.
130809f0800SChristian Ehrhardt */
131809f0800SChristian Ehrhardtstruct ibv_mr *
132809f0800SChristian Ehrhardtmlx5_mp2mr(struct ibv_pd *pd, struct rte_mempool *mp)
133809f0800SChristian Ehrhardt{
134809f0800SChristian Ehrhardt	const struct rte_memseg *ms = rte_eal_get_physmem_layout();
135809f0800SChristian Ehrhardt	uintptr_t start;
136809f0800SChristian Ehrhardt	uintptr_t end;
137809f0800SChristian Ehrhardt	unsigned int i;
138809f0800SChristian Ehrhardt
139809f0800SChristian Ehrhardt	if (mlx5_check_mempool(mp, &start, &end) != 0) {
140809f0800SChristian Ehrhardt		ERROR("mempool %p: not virtually contiguous",
141809f0800SChristian Ehrhardt		      (void *)mp);
142809f0800SChristian Ehrhardt		return NULL;
143809f0800SChristian Ehrhardt	}
144809f0800SChristian Ehrhardt
145809f0800SChristian Ehrhardt	DEBUG("mempool %p area start=%p end=%p size=%zu",
146809f0800SChristian Ehrhardt	      (void *)mp, (void *)start, (void *)end,
147809f0800SChristian Ehrhardt	      (size_t)(end - start));
148809f0800SChristian Ehrhardt	/* Round start and end to page boundary if found in memory segments. */
149809f0800SChristian Ehrhardt	for (i = 0; (i < RTE_MAX_MEMSEG) && (ms[i].addr != NULL); ++i) {
150809f0800SChristian Ehrhardt		uintptr_t addr = (uintptr_t)ms[i].addr;
151809f0800SChristian Ehrhardt		size_t len = ms[i].len;
152809f0800SChristian Ehrhardt		unsigned int align = ms[i].hugepage_sz;
153809f0800SChristian Ehrhardt
154809f0800SChristian Ehrhardt		if ((start > addr) && (start < addr + len))
155809f0800SChristian Ehrhardt			start = RTE_ALIGN_FLOOR(start, align);
156809f0800SChristian Ehrhardt		if ((end > addr) && (end < addr + len))
157809f0800SChristian Ehrhardt			end = RTE_ALIGN_CEIL(end, align);
158809f0800SChristian Ehrhardt	}
159809f0800SChristian Ehrhardt	DEBUG("mempool %p using start=%p end=%p size=%zu for MR",
160809f0800SChristian Ehrhardt	      (void *)mp, (void *)start, (void *)end,
161809f0800SChristian Ehrhardt	      (size_t)(end - start));
162809f0800SChristian Ehrhardt	return ibv_reg_mr(pd,
163809f0800SChristian Ehrhardt			  (void *)start,
164809f0800SChristian Ehrhardt			  end - start,
165809f0800SChristian Ehrhardt			  IBV_ACCESS_LOCAL_WRITE);
166809f0800SChristian Ehrhardt}
167809f0800SChristian Ehrhardt
168809f0800SChristian Ehrhardt/**
169809f0800SChristian Ehrhardt * Register a Memory Region (MR) <-> Memory Pool (MP) association in
170809f0800SChristian Ehrhardt * txq->mp2mr[]. If mp2mr[] is full, remove an entry first.
171809f0800SChristian Ehrhardt *
172809f0800SChristian Ehrhardt * This function should only be called by txq_mp2mr().
173809f0800SChristian Ehrhardt *
174809f0800SChristian Ehrhardt * @param txq
175809f0800SChristian Ehrhardt *   Pointer to TX queue structure.
176809f0800SChristian Ehrhardt * @param[in] mp
177809f0800SChristian Ehrhardt *   Memory Pool for which a Memory Region lkey must be returned.
178809f0800SChristian Ehrhardt * @param idx
179809f0800SChristian Ehrhardt *   Index of the next available entry.
180809f0800SChristian Ehrhardt *
181809f0800SChristian Ehrhardt * @return
182809f0800SChristian Ehrhardt *   mr->lkey on success, (uint32_t)-1 on failure.
183809f0800SChristian Ehrhardt */
184809f0800SChristian Ehrhardtuint32_t
185809f0800SChristian Ehrhardttxq_mp2mr_reg(struct txq *txq, struct rte_mempool *mp, unsigned int idx)
186809f0800SChristian Ehrhardt{
187809f0800SChristian Ehrhardt	struct txq_ctrl *txq_ctrl = container_of(txq, struct txq_ctrl, txq);
188809f0800SChristian Ehrhardt	struct ibv_mr *mr;
189809f0800SChristian Ehrhardt
190809f0800SChristian Ehrhardt	/* Add a new entry, register MR first. */
191809f0800SChristian Ehrhardt	DEBUG("%p: discovered new memory pool \"%s\" (%p)",
192809f0800SChristian Ehrhardt	      (void *)txq_ctrl, mp->name, (void *)mp);
193809f0800SChristian Ehrhardt	mr = mlx5_mp2mr(txq_ctrl->priv->pd, mp);
194809f0800SChristian Ehrhardt	if (unlikely(mr == NULL)) {
195809f0800SChristian Ehrhardt		DEBUG("%p: unable to configure MR, ibv_reg_mr() failed.",
196809f0800SChristian Ehrhardt		      (void *)txq_ctrl);
197809f0800SChristian Ehrhardt		return (uint32_t)-1;
198809f0800SChristian Ehrhardt	}
199809f0800SChristian Ehrhardt	if (unlikely(idx == RTE_DIM(txq_ctrl->txq.mp2mr))) {
200809f0800SChristian Ehrhardt		/* Table is full, remove oldest entry. */
201809f0800SChristian Ehrhardt		DEBUG("%p: MR <-> MP table full, dropping oldest entry.",
202809f0800SChristian Ehrhardt		      (void *)txq_ctrl);
203809f0800SChristian Ehrhardt		--idx;
204809f0800SChristian Ehrhardt		claim_zero(ibv_dereg_mr(txq_ctrl->txq.mp2mr[0].mr));
205809f0800SChristian Ehrhardt		memmove(&txq_ctrl->txq.mp2mr[0], &txq_ctrl->txq.mp2mr[1],
206809f0800SChristian Ehrhardt			(sizeof(txq_ctrl->txq.mp2mr) -
207809f0800SChristian Ehrhardt			 sizeof(txq_ctrl->txq.mp2mr[0])));
208809f0800SChristian Ehrhardt	}
209809f0800SChristian Ehrhardt	/* Store the new entry. */
210809f0800SChristian Ehrhardt	txq_ctrl->txq.mp2mr[idx].mp = mp;
211809f0800SChristian Ehrhardt	txq_ctrl->txq.mp2mr[idx].mr = mr;
212809f0800SChristian Ehrhardt	txq_ctrl->txq.mp2mr[idx].lkey = htonl(mr->lkey);
213809f0800SChristian Ehrhardt	DEBUG("%p: new MR lkey for MP \"%s\" (%p): 0x%08" PRIu32,
214809f0800SChristian Ehrhardt	      (void *)txq_ctrl, mp->name, (void *)mp,
215809f0800SChristian Ehrhardt	      txq_ctrl->txq.mp2mr[idx].lkey);
216809f0800SChristian Ehrhardt	return txq_ctrl->txq.mp2mr[idx].lkey;
217809f0800SChristian Ehrhardt}
218809f0800SChristian Ehrhardt
219809f0800SChristian Ehrhardtstruct txq_mp2mr_mbuf_check_data {
220809f0800SChristian Ehrhardt	int ret;
221809f0800SChristian Ehrhardt};
222809f0800SChristian Ehrhardt
223809f0800SChristian Ehrhardt/**
224809f0800SChristian Ehrhardt * Callback function for rte_mempool_obj_iter() to check whether a given
225809f0800SChristian Ehrhardt * mempool object looks like a mbuf.
226809f0800SChristian Ehrhardt *
227809f0800SChristian Ehrhardt * @param[in] mp
228809f0800SChristian Ehrhardt *   The mempool pointer
229809f0800SChristian Ehrhardt * @param[in] arg
230809f0800SChristian Ehrhardt *   Context data (struct txq_mp2mr_mbuf_check_data). Contains the
231809f0800SChristian Ehrhardt *   return value.
232809f0800SChristian Ehrhardt * @param[in] obj
233809f0800SChristian Ehrhardt *   Object address.
234809f0800SChristian Ehrhardt * @param index
235809f0800SChristian Ehrhardt *   Object index, unused.
236809f0800SChristian Ehrhardt */
237809f0800SChristian Ehrhardtstatic void
238809f0800SChristian Ehrhardttxq_mp2mr_mbuf_check(struct rte_mempool *mp, void *arg, void *obj,
239809f0800SChristian Ehrhardt	uint32_t index __rte_unused)
240809f0800SChristian Ehrhardt{
241809f0800SChristian Ehrhardt	struct txq_mp2mr_mbuf_check_data *data = arg;
242809f0800SChristian Ehrhardt	struct rte_mbuf *buf = obj;
243809f0800SChristian Ehrhardt
244809f0800SChristian Ehrhardt	/*
245809f0800SChristian Ehrhardt	 * Check whether mbuf structure fits element size and whether mempool
246809f0800SChristian Ehrhardt	 * pointer is valid.
247809f0800SChristian Ehrhardt	 */
248809f0800SChristian Ehrhardt	if (sizeof(*buf) > mp->elt_size || buf->pool != mp)
249809f0800SChristian Ehrhardt		data->ret = -1;
250809f0800SChristian Ehrhardt}
251809f0800SChristian Ehrhardt
252809f0800SChristian Ehrhardt/**
253809f0800SChristian Ehrhardt * Iterator function for rte_mempool_walk() to register existing mempools and
254809f0800SChristian Ehrhardt * fill the MP to MR cache of a TX queue.
255809f0800SChristian Ehrhardt *
256809f0800SChristian Ehrhardt * @param[in] mp
257809f0800SChristian Ehrhardt *   Memory Pool to register.
258809f0800SChristian Ehrhardt * @param *arg
259809f0800SChristian Ehrhardt *   Pointer to TX queue structure.
260809f0800SChristian Ehrhardt */
261809f0800SChristian Ehrhardtvoid
262809f0800SChristian Ehrhardttxq_mp2mr_iter(struct rte_mempool *mp, void *arg)
263809f0800SChristian Ehrhardt{
264809f0800SChristian Ehrhardt	struct txq_ctrl *txq_ctrl = arg;
265809f0800SChristian Ehrhardt	struct txq_mp2mr_mbuf_check_data data = {
266809f0800SChristian Ehrhardt		.ret = 0,
267809f0800SChristian Ehrhardt	};
268809f0800SChristian Ehrhardt	unsigned int i;
269809f0800SChristian Ehrhardt
270809f0800SChristian Ehrhardt	/* Register mempool only if the first element looks like a mbuf. */
271809f0800SChristian Ehrhardt	if (rte_mempool_obj_iter(mp, txq_mp2mr_mbuf_check, &data) == 0 ||
272809f0800SChristian Ehrhardt			data.ret == -1)
273809f0800SChristian Ehrhardt		return;
274809f0800SChristian Ehrhardt	for (i = 0; (i != RTE_DIM(txq_ctrl->txq.mp2mr)); ++i) {
275809f0800SChristian Ehrhardt		if (unlikely(txq_ctrl->txq.mp2mr[i].mp == NULL)) {
276809f0800SChristian Ehrhardt			/* Unknown MP, add a new MR for it. */
277809f0800SChristian Ehrhardt			break;
278809f0800SChristian Ehrhardt		}
279809f0800SChristian Ehrhardt		if (txq_ctrl->txq.mp2mr[i].mp == mp)
280809f0800SChristian Ehrhardt			return;
281809f0800SChristian Ehrhardt	}
282809f0800SChristian Ehrhardt	txq_mp2mr_reg(&txq_ctrl->txq, mp, i);
283809f0800SChristian Ehrhardt}
284