1/*-
2 *   BSD LICENSE
3 *
4 *   Copyright 2016 6WIND S.A.
5 *   Copyright 2016 Mellanox.
6 *
7 *   Redistribution and use in source and binary forms, with or without
8 *   modification, are permitted provided that the following conditions
9 *   are met:
10 *
11 *     * Redistributions of source code must retain the above copyright
12 *       notice, this list of conditions and the following disclaimer.
13 *     * Redistributions in binary form must reproduce the above copyright
14 *       notice, this list of conditions and the following disclaimer in
15 *       the documentation and/or other materials provided with the
16 *       distribution.
17 *     * Neither the name of 6WIND S.A. nor the names of its
18 *       contributors may be used to endorse or promote products derived
19 *       from this software without specific prior written permission.
20 *
21 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34/* Verbs header. */
35/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
36#ifdef PEDANTIC
37#pragma GCC diagnostic ignored "-Wpedantic"
38#endif
39#include <infiniband/verbs.h>
40#ifdef PEDANTIC
41#pragma GCC diagnostic error "-Wpedantic"
42#endif
43
44/* DPDK headers don't like -pedantic. */
45#ifdef PEDANTIC
46#pragma GCC diagnostic ignored "-Wpedantic"
47#endif
48#include <rte_mempool.h>
49#ifdef PEDANTIC
50#pragma GCC diagnostic error "-Wpedantic"
51#endif
52
53#include "mlx5.h"
54#include "mlx5_rxtx.h"
55
56struct mlx5_check_mempool_data {
57	int ret;
58	char *start;
59	char *end;
60};
61
62/* Called by mlx5_check_mempool() when iterating the memory chunks. */
63static void
64mlx5_check_mempool_cb(struct rte_mempool *mp,
65		      void *opaque, struct rte_mempool_memhdr *memhdr,
66		      unsigned int mem_idx)
67{
68	struct mlx5_check_mempool_data *data = opaque;
69
70	(void)mp;
71	(void)mem_idx;
72
73	/* It already failed, skip the next chunks. */
74	if (data->ret != 0)
75		return;
76	/* It is the first chunk. */
77	if (data->start == NULL && data->end == NULL) {
78		data->start = memhdr->addr;
79		data->end = data->start + memhdr->len;
80		return;
81	}
82	if (data->end == memhdr->addr) {
83		data->end += memhdr->len;
84		return;
85	}
86	if (data->start == (char *)memhdr->addr + memhdr->len) {
87		data->start -= memhdr->len;
88		return;
89	}
90	/* Error, mempool is not virtually contiguous. */
91	data->ret = -1;
92}
93
94/**
95 * Check if a mempool can be used: it must be virtually contiguous.
96 *
97 * @param[in] mp
98 *   Pointer to memory pool.
99 * @param[out] start
100 *   Pointer to the start address of the mempool virtual memory area
101 * @param[out] end
102 *   Pointer to the end address of the mempool virtual memory area
103 *
104 * @return
105 *   0 on success (mempool is virtually contiguous), -1 on error.
106 */
107static int mlx5_check_mempool(struct rte_mempool *mp, uintptr_t *start,
108	uintptr_t *end)
109{
110	struct mlx5_check_mempool_data data;
111
112	memset(&data, 0, sizeof(data));
113	rte_mempool_mem_iter(mp, mlx5_check_mempool_cb, &data);
114	*start = (uintptr_t)data.start;
115	*end = (uintptr_t)data.end;
116
117	return data.ret;
118}
119
120/**
121 * Register mempool as a memory region.
122 *
123 * @param pd
124 *   Pointer to protection domain.
125 * @param mp
126 *   Pointer to memory pool.
127 *
128 * @return
129 *   Memory region pointer, NULL in case of error.
130 */
131struct ibv_mr *
132mlx5_mp2mr(struct ibv_pd *pd, struct rte_mempool *mp)
133{
134	const struct rte_memseg *ms = rte_eal_get_physmem_layout();
135	uintptr_t start;
136	uintptr_t end;
137	unsigned int i;
138
139	if (mlx5_check_mempool(mp, &start, &end) != 0) {
140		ERROR("mempool %p: not virtually contiguous",
141		      (void *)mp);
142		return NULL;
143	}
144
145	DEBUG("mempool %p area start=%p end=%p size=%zu",
146	      (void *)mp, (void *)start, (void *)end,
147	      (size_t)(end - start));
148	/* Round start and end to page boundary if found in memory segments. */
149	for (i = 0; (i < RTE_MAX_MEMSEG) && (ms[i].addr != NULL); ++i) {
150		uintptr_t addr = (uintptr_t)ms[i].addr;
151		size_t len = ms[i].len;
152		unsigned int align = ms[i].hugepage_sz;
153
154		if ((start > addr) && (start < addr + len))
155			start = RTE_ALIGN_FLOOR(start, align);
156		if ((end > addr) && (end < addr + len))
157			end = RTE_ALIGN_CEIL(end, align);
158	}
159	DEBUG("mempool %p using start=%p end=%p size=%zu for MR",
160	      (void *)mp, (void *)start, (void *)end,
161	      (size_t)(end - start));
162	return ibv_reg_mr(pd,
163			  (void *)start,
164			  end - start,
165			  IBV_ACCESS_LOCAL_WRITE);
166}
167
168/**
169 * Register a Memory Region (MR) <-> Memory Pool (MP) association in
170 * txq->mp2mr[]. If mp2mr[] is full, remove an entry first.
171 *
172 * This function should only be called by txq_mp2mr().
173 *
174 * @param txq
175 *   Pointer to TX queue structure.
176 * @param[in] mp
177 *   Memory Pool for which a Memory Region lkey must be returned.
178 * @param idx
179 *   Index of the next available entry.
180 *
181 * @return
182 *   mr->lkey on success, (uint32_t)-1 on failure.
183 */
184uint32_t
185txq_mp2mr_reg(struct txq *txq, struct rte_mempool *mp, unsigned int idx)
186{
187	struct txq_ctrl *txq_ctrl = container_of(txq, struct txq_ctrl, txq);
188	struct ibv_mr *mr;
189
190	/* Add a new entry, register MR first. */
191	DEBUG("%p: discovered new memory pool \"%s\" (%p)",
192	      (void *)txq_ctrl, mp->name, (void *)mp);
193	mr = mlx5_mp2mr(txq_ctrl->priv->pd, mp);
194	if (unlikely(mr == NULL)) {
195		DEBUG("%p: unable to configure MR, ibv_reg_mr() failed.",
196		      (void *)txq_ctrl);
197		return (uint32_t)-1;
198	}
199	if (unlikely(idx == RTE_DIM(txq_ctrl->txq.mp2mr))) {
200		/* Table is full, remove oldest entry. */
201		DEBUG("%p: MR <-> MP table full, dropping oldest entry.",
202		      (void *)txq_ctrl);
203		--idx;
204		claim_zero(ibv_dereg_mr(txq_ctrl->txq.mp2mr[0].mr));
205		memmove(&txq_ctrl->txq.mp2mr[0], &txq_ctrl->txq.mp2mr[1],
206			(sizeof(txq_ctrl->txq.mp2mr) -
207			 sizeof(txq_ctrl->txq.mp2mr[0])));
208	}
209	/* Store the new entry. */
210	txq_ctrl->txq.mp2mr[idx].mp = mp;
211	txq_ctrl->txq.mp2mr[idx].mr = mr;
212	txq_ctrl->txq.mp2mr[idx].lkey = htonl(mr->lkey);
213	DEBUG("%p: new MR lkey for MP \"%s\" (%p): 0x%08" PRIu32,
214	      (void *)txq_ctrl, mp->name, (void *)mp,
215	      txq_ctrl->txq.mp2mr[idx].lkey);
216	return txq_ctrl->txq.mp2mr[idx].lkey;
217}
218
219struct txq_mp2mr_mbuf_check_data {
220	int ret;
221};
222
223/**
224 * Callback function for rte_mempool_obj_iter() to check whether a given
225 * mempool object looks like a mbuf.
226 *
227 * @param[in] mp
228 *   The mempool pointer
229 * @param[in] arg
230 *   Context data (struct txq_mp2mr_mbuf_check_data). Contains the
231 *   return value.
232 * @param[in] obj
233 *   Object address.
234 * @param index
235 *   Object index, unused.
236 */
237static void
238txq_mp2mr_mbuf_check(struct rte_mempool *mp, void *arg, void *obj,
239	uint32_t index __rte_unused)
240{
241	struct txq_mp2mr_mbuf_check_data *data = arg;
242	struct rte_mbuf *buf = obj;
243
244	/*
245	 * Check whether mbuf structure fits element size and whether mempool
246	 * pointer is valid.
247	 */
248	if (sizeof(*buf) > mp->elt_size || buf->pool != mp)
249		data->ret = -1;
250}
251
252/**
253 * Iterator function for rte_mempool_walk() to register existing mempools and
254 * fill the MP to MR cache of a TX queue.
255 *
256 * @param[in] mp
257 *   Memory Pool to register.
258 * @param *arg
259 *   Pointer to TX queue structure.
260 */
261void
262txq_mp2mr_iter(struct rte_mempool *mp, void *arg)
263{
264	struct txq_ctrl *txq_ctrl = arg;
265	struct txq_mp2mr_mbuf_check_data data = {
266		.ret = 0,
267	};
268	unsigned int i;
269
270	/* Register mempool only if the first element looks like a mbuf. */
271	if (rte_mempool_obj_iter(mp, txq_mp2mr_mbuf_check, &data) == 0 ||
272			data.ret == -1)
273		return;
274	for (i = 0; (i != RTE_DIM(txq_ctrl->txq.mp2mr)); ++i) {
275		if (unlikely(txq_ctrl->txq.mp2mr[i].mp == NULL)) {
276			/* Unknown MP, add a new MR for it. */
277			break;
278		}
279		if (txq_ctrl->txq.mp2mr[i].mp == mp)
280			return;
281	}
282	txq_mp2mr_reg(&txq_ctrl->txq, mp, i);
283}
284