ctx.c revision aa97dd1c
1/*
2 * Copyright (c) 2016  Intel Corporation.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 *     http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#include <string.h>
17#include <rte_malloc.h>
18#include <rte_errno.h>
19#include <rte_ethdev.h>
20#include <rte_ip.h>
21
22#include "stream.h"
23#include "misc.h"
24
25#define	LPORT_START	0x8000
26#define	LPORT_END	MAX_PORT_NUM
27
28#define	LPORT_START_BLK	PORT_BLK(LPORT_START)
29#define	LPORT_END_BLK	PORT_BLK(LPORT_END)
30
31const struct in6_addr tle_ipv6_any = IN6ADDR_ANY_INIT;
32const struct in6_addr tle_ipv6_none = {
33	{
34		.__u6_addr32 = {
35			UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX
36		},
37	},
38};
39
40struct stream_ops tle_stream_ops[TLE_PROTO_NUM] = {};
41
42static int
43check_dev_prm(const struct tle_dev_param *dev_prm)
44{
45	/* no valid IPv4/IPv6 addresses provided. */
46	if (dev_prm->local_addr4.s_addr == INADDR_ANY &&
47			memcmp(&dev_prm->local_addr6, &tle_ipv6_any,
48			sizeof(tle_ipv6_any)) == 0)
49		return -EINVAL;
50
51	if (dev_prm->bl4.nb_port > UINT16_MAX ||
52			(dev_prm->bl4.nb_port != 0 &&
53			dev_prm->bl4.port == NULL))
54		return -EINVAL;
55
56	if (dev_prm->bl6.nb_port > UINT16_MAX ||
57			(dev_prm->bl6.nb_port != 0 &&
58			dev_prm->bl6.port == NULL))
59		return -EINVAL;
60
61	return 0;
62}
63
64static int
65check_ctx_prm(const struct tle_ctx_param *prm)
66{
67	if (prm->proto >= TLE_PROTO_NUM)
68		return -EINVAL;
69	return 0;
70}
71
72struct tle_ctx *
73tle_ctx_create(const struct tle_ctx_param *ctx_prm)
74{
75	struct tle_ctx *ctx;
76	size_t sz;
77	uint32_t i;
78	int32_t rc;
79
80	if (ctx_prm == NULL || check_ctx_prm(ctx_prm) != 0) {
81		rte_errno = EINVAL;
82		return NULL;
83	}
84
85	sz = sizeof(*ctx);
86	ctx = rte_zmalloc_socket(NULL, sz, RTE_CACHE_LINE_SIZE,
87		ctx_prm->socket_id);
88	if (ctx == NULL) {
89		UDP_LOG(ERR, "allocation of %zu bytes for new ctx "
90			"on socket %d failed\n",
91			sz, ctx_prm->socket_id);
92		return NULL;
93	}
94
95	ctx->prm = *ctx_prm;
96
97	rc = tle_stream_ops[ctx_prm->proto].init_streams(ctx);
98	if (rc != 0) {
99		UDP_LOG(ERR, "init_streams(ctx=%p, proto=%u) failed "
100			"with error code: %d;\n",
101			ctx, ctx_prm->proto, rc);
102		tle_ctx_destroy(ctx);
103		rte_errno = -rc;
104		return NULL;
105	}
106
107	for (i = 0; i != RTE_DIM(ctx->use); i++)
108		tle_pbm_init(ctx->use + i, LPORT_START_BLK);
109
110	ctx->streams.nb_free = ctx->prm.max_streams;
111	return ctx;
112}
113
114void
115tle_ctx_destroy(struct tle_ctx *ctx)
116{
117	uint32_t i;
118
119	if (ctx == NULL) {
120		rte_errno = EINVAL;
121		return;
122	}
123
124	for (i = 0; i != RTE_DIM(ctx->dev); i++)
125		tle_del_dev(ctx->dev + i);
126
127	tle_stream_ops[ctx->prm.proto].fini_streams(ctx);
128	rte_free(ctx);
129}
130
131void
132tle_ctx_invalidate(struct tle_ctx *ctx)
133{
134	RTE_SET_USED(ctx);
135}
136
137static void
138fill_pbm(struct tle_pbm *pbm, const struct tle_bl_port *blp)
139{
140	uint32_t i;
141
142	for (i = 0; i != blp->nb_port; i++)
143		tle_pbm_set(pbm, blp->port[i]);
144}
145
146static int
147init_dev_proto(struct tle_dev *dev, uint32_t idx, int32_t socket_id,
148	const struct tle_bl_port *blp)
149{
150	size_t sz;
151
152	sz = sizeof(*dev->dp[idx]);
153	dev->dp[idx] = rte_zmalloc_socket(NULL, sz, RTE_CACHE_LINE_SIZE,
154		socket_id);
155
156	if (dev->dp[idx] == NULL) {
157		UDP_LOG(ERR, "allocation of %zu bytes on "
158			"socket %d for %u-th device failed\n",
159			sz, socket_id, idx);
160		return ENOMEM;
161	}
162
163	tle_pbm_init(&dev->dp[idx]->use, LPORT_START_BLK);
164	fill_pbm(&dev->dp[idx]->use, blp);
165	return 0;
166}
167
168static struct tle_dev *
169find_free_dev(struct tle_ctx *ctx)
170{
171	uint32_t i;
172
173	if (ctx->nb_dev < RTE_DIM(ctx->dev)) {
174		for (i = 0; i != RTE_DIM(ctx->dev); i++) {
175			if (ctx->dev[i].ctx != ctx)
176				return ctx->dev + i;
177		}
178	}
179
180	rte_errno = ENODEV;
181	return NULL;
182}
183
184struct tle_dev *
185tle_add_dev(struct tle_ctx *ctx, const struct tle_dev_param *dev_prm)
186{
187	int32_t rc;
188	struct tle_dev *dev;
189
190	if (ctx == NULL || dev_prm == NULL || check_dev_prm(dev_prm) != 0) {
191		rte_errno = EINVAL;
192		return NULL;
193	}
194
195	dev = find_free_dev(ctx);
196	if (dev == NULL)
197		return NULL;
198	rc = 0;
199
200	/* device can handle IPv4 traffic */
201	if (dev_prm->local_addr4.s_addr != INADDR_ANY) {
202		rc = init_dev_proto(dev, TLE_V4, ctx->prm.socket_id,
203			&dev_prm->bl4);
204		if (rc == 0)
205			fill_pbm(&ctx->use[TLE_V4], &dev_prm->bl4);
206	}
207
208	/* device can handle IPv6 traffic */
209	if (rc == 0 && memcmp(&dev_prm->local_addr6, &tle_ipv6_any,
210			sizeof(tle_ipv6_any)) != 0) {
211		rc = init_dev_proto(dev, TLE_V6, ctx->prm.socket_id,
212			&dev_prm->bl6);
213		if (rc == 0)
214			fill_pbm(&ctx->use[TLE_V6], &dev_prm->bl6);
215	}
216
217	if (rc != 0) {
218		/* cleanup and return an error. */
219		rte_free(dev->dp[TLE_V4]);
220		rte_free(dev->dp[TLE_V6]);
221		rte_errno = rc;
222		return NULL;
223	}
224
225	/* setup RX data. */
226	if (dev_prm->local_addr4.s_addr != INADDR_ANY &&
227			(dev_prm->rx_offload & DEV_RX_OFFLOAD_IPV4_CKSUM) == 0)
228		dev->rx.ol_flags[TLE_V4] |= PKT_RX_IP_CKSUM_BAD;
229
230	if (((dev_prm->rx_offload & DEV_RX_OFFLOAD_UDP_CKSUM) == 0 &&
231			ctx->prm.proto == TLE_PROTO_UDP) ||
232			((dev_prm->rx_offload &
233			DEV_RX_OFFLOAD_TCP_CKSUM) == 0 &&
234			ctx->prm.proto == TLE_PROTO_TCP)) {
235		dev->rx.ol_flags[TLE_V4] |= PKT_RX_L4_CKSUM_BAD;
236		dev->rx.ol_flags[TLE_V6] |= PKT_RX_L4_CKSUM_BAD;
237	}
238
239	/* setup TX data. */
240	tle_dring_reset(&dev->tx.dr);
241
242	if ((dev_prm->tx_offload & DEV_TX_OFFLOAD_UDP_CKSUM) != 0 &&
243			ctx->prm.proto == TLE_PROTO_UDP) {
244		dev->tx.ol_flags[TLE_V4] |= PKT_TX_IPV4 | PKT_TX_UDP_CKSUM;
245		dev->tx.ol_flags[TLE_V6] |= PKT_TX_IPV6 | PKT_TX_UDP_CKSUM;
246	} else if ((dev_prm->tx_offload & DEV_TX_OFFLOAD_TCP_CKSUM) != 0 &&
247			ctx->prm.proto == TLE_PROTO_TCP) {
248		dev->tx.ol_flags[TLE_V4] |= PKT_TX_IPV4 | PKT_TX_TCP_CKSUM;
249		dev->tx.ol_flags[TLE_V6] |= PKT_TX_IPV6 | PKT_TX_TCP_CKSUM;
250	}
251
252	if ((dev_prm->tx_offload & DEV_TX_OFFLOAD_IPV4_CKSUM) != 0)
253		dev->tx.ol_flags[TLE_V4] |= PKT_TX_IPV4 | PKT_TX_IP_CKSUM;
254
255	dev->prm = *dev_prm;
256	dev->ctx = ctx;
257	ctx->nb_dev++;
258
259	return dev;
260}
261
262static void
263empty_dring(struct tle_dring *dr, uint32_t proto)
264{
265	uint32_t i, k, n;
266	struct tle_stream *s;
267	struct rte_mbuf *pkt[MAX_PKT_BURST];
268	struct tle_drb *drb[MAX_PKT_BURST];
269
270	do {
271		k = RTE_DIM(drb);
272		n = tle_dring_sc_dequeue(dr, (const void **)(uintptr_t)pkt,
273			RTE_DIM(pkt), drb, &k);
274
275		/* free mbufs */
276		for (i = 0; i != n; i++)
277			rte_pktmbuf_free(pkt[i]);
278		/* free drbs */
279		for (i = 0; i != k; i++) {
280			s = drb[i]->udata;
281			tle_stream_ops[proto].free_drbs(s, drb + i, 1);
282		}
283	} while (n != 0);
284}
285
286int
287tle_del_dev(struct tle_dev *dev)
288{
289	uint32_t p;
290	struct tle_ctx *ctx;
291
292	if (dev == NULL || dev->ctx == NULL)
293		return -EINVAL;
294
295	ctx = dev->ctx;
296	p = dev - ctx->dev;
297
298	if (p >= RTE_DIM(ctx->dev) ||
299			(dev->dp[TLE_V4] == NULL &&
300			dev->dp[TLE_V6] == NULL))
301		return -EINVAL;
302
303	/* emtpy TX queues. */
304	empty_dring(&dev->tx.dr, ctx->prm.proto);
305
306	rte_free(dev->dp[TLE_V4]);
307	rte_free(dev->dp[TLE_V6]);
308	memset(dev, 0, sizeof(*dev));
309	ctx->nb_dev--;
310	return 0;
311}
312
313static struct tle_dev *
314find_ipv4_dev(struct tle_ctx *ctx, const struct in_addr *addr)
315{
316	uint32_t i;
317
318	for (i = 0; i != RTE_DIM(ctx->dev); i++) {
319		if (ctx->dev[i].prm.local_addr4.s_addr == addr->s_addr &&
320				ctx->dev[i].dp[TLE_V4] != NULL)
321			return ctx->dev + i;
322	}
323
324	return NULL;
325}
326
327static struct tle_dev *
328find_ipv6_dev(struct tle_ctx *ctx, const struct in6_addr *addr)
329{
330	uint32_t i;
331
332	for (i = 0; i != RTE_DIM(ctx->dev); i++) {
333		if (memcmp(&ctx->dev[i].prm.local_addr6, addr,
334				sizeof(*addr)) == 0 &&
335				ctx->dev[i].dp[TLE_V6] != NULL)
336			return ctx->dev + i;
337	}
338
339	return NULL;
340}
341
342static int
343stream_fill_dev(struct tle_ctx *ctx, struct tle_stream *s,
344	const struct sockaddr *addr)
345{
346	struct tle_dev *dev;
347	struct tle_pbm *pbm;
348	const struct sockaddr_in *lin4;
349	const struct sockaddr_in6 *lin6;
350	uint32_t i, p, sp, t;
351
352	if (addr->sa_family == AF_INET) {
353		lin4 = (const struct sockaddr_in *)addr;
354		t = TLE_V4;
355		p = lin4->sin_port;
356	} else if (addr->sa_family == AF_INET6) {
357		lin6 = (const struct sockaddr_in6 *)addr;
358		t = TLE_V6;
359		p = lin6->sin6_port;
360	} else
361		return EINVAL;
362
363	p = ntohs(p);
364
365	/* if local address is not wildcard, find device it belongs to. */
366	if (t == TLE_V4 && lin4->sin_addr.s_addr != INADDR_ANY) {
367		dev = find_ipv4_dev(ctx, &lin4->sin_addr);
368		if (dev == NULL)
369			return ENODEV;
370	} else if (t == TLE_V6 && memcmp(&tle_ipv6_any, &lin6->sin6_addr,
371			sizeof(tle_ipv6_any)) != 0) {
372		dev = find_ipv6_dev(ctx, &lin6->sin6_addr);
373		if (dev == NULL)
374			return ENODEV;
375	} else
376		dev = NULL;
377
378	if (dev != NULL)
379		pbm = &dev->dp[t]->use;
380	else
381		pbm = &ctx->use[t];
382
383	/* try to acquire local port number. */
384	if (p == 0) {
385		p = tle_pbm_find_range(pbm, pbm->blk, LPORT_END_BLK);
386		if (p == 0 && pbm->blk > LPORT_START_BLK)
387			p = tle_pbm_find_range(pbm, LPORT_START_BLK, pbm->blk);
388	} else if (tle_pbm_check(pbm, p) != 0)
389		return EEXIST;
390
391	if (p == 0)
392		return ENFILE;
393
394	/* fill socket's dst port and type */
395
396	sp = htons(p);
397	s->type = t;
398	s->port.dst = sp;
399
400	/* mark port as in-use */
401
402	tle_pbm_set(&ctx->use[t], p);
403	if (dev != NULL) {
404		tle_pbm_set(pbm, p);
405		dev->dp[t]->streams[sp] = s;
406	} else {
407		for (i = 0; i != RTE_DIM(ctx->dev); i++) {
408			if (ctx->dev[i].dp[t] != NULL) {
409				tle_pbm_set(&ctx->dev[i].dp[t]->use, p);
410				ctx->dev[i].dp[t]->streams[sp] = s;
411			}
412		}
413	}
414
415	return 0;
416}
417
418static int
419stream_clear_dev(struct tle_ctx *ctx, const struct tle_stream *s)
420{
421	struct tle_dev *dev;
422	uint32_t i, p, sp, t;
423
424	t = s->type;
425	sp = s->port.dst;
426	p = ntohs(sp);
427
428	/* if local address is not wildcard, find device it belongs to. */
429	if (t == TLE_V4 && s->ipv4.addr.dst != INADDR_ANY) {
430		dev = find_ipv4_dev(ctx,
431			(const struct in_addr *)&s->ipv4.addr.dst);
432		if (dev == NULL)
433			return ENODEV;
434	} else if (t == TLE_V6 && memcmp(&tle_ipv6_any, &s->ipv6.addr.dst,
435			sizeof(tle_ipv6_any)) != 0) {
436		dev = find_ipv6_dev(ctx,
437			(const struct in6_addr *)&s->ipv6.addr.dst);
438		if (dev == NULL)
439			return ENODEV;
440	} else
441		dev = NULL;
442
443	tle_pbm_clear(&ctx->use[t], p);
444	if (dev != NULL) {
445		if (dev->dp[t]->streams[sp] == s) {
446			tle_pbm_clear(&dev->dp[t]->use, p);
447			dev->dp[t]->streams[sp] = NULL;
448		}
449	} else {
450		for (i = 0; i != RTE_DIM(ctx->dev); i++) {
451			if (ctx->dev[i].dp[t] != NULL &&
452					ctx->dev[i].dp[t]->streams[sp] == s) {
453				tle_pbm_clear(&ctx->dev[i].dp[t]->use, p);
454				ctx->dev[i].dp[t]->streams[sp] = NULL;
455			}
456		}
457	}
458
459	return 0;
460}
461
462static void
463fill_ipv4_am(const struct sockaddr_in *in, uint32_t *addr, uint32_t *mask)
464{
465	*addr = in->sin_addr.s_addr;
466	*mask = (*addr == INADDR_ANY) ? INADDR_ANY : INADDR_NONE;
467}
468
469static void
470fill_ipv6_am(const struct sockaddr_in6 *in, rte_xmm_t *addr, rte_xmm_t *mask)
471{
472	const struct in6_addr *pm;
473
474	memcpy(addr, &in->sin6_addr, sizeof(*addr));
475	if (memcmp(&tle_ipv6_any, addr, sizeof(*addr)) == 0)
476		pm = &tle_ipv6_any;
477	else
478		pm = &tle_ipv6_none;
479
480	memcpy(mask, pm, sizeof(*mask));
481}
482
483int
484stream_fill_ctx(struct tle_ctx *ctx, struct tle_stream *s,
485	const struct sockaddr *laddr, const struct sockaddr *raddr)
486{
487	const struct sockaddr_in *rin;
488	int32_t rc;
489
490	/* setup ports and port mask fields (except dst port). */
491	rin = (const struct sockaddr_in *)raddr;
492	s->port.src = rin->sin_port;
493	s->pmsk.src = (s->port.src == 0) ? 0 : UINT16_MAX;
494	s->pmsk.dst = UINT16_MAX;
495
496	/* setup src and dst addresses. */
497	if (laddr->sa_family == AF_INET) {
498		fill_ipv4_am((const struct sockaddr_in *)laddr,
499			&s->ipv4.addr.dst, &s->ipv4.mask.dst);
500		fill_ipv4_am((const struct sockaddr_in *)raddr,
501			&s->ipv4.addr.src, &s->ipv4.mask.src);
502	} else if (laddr->sa_family == AF_INET6) {
503		fill_ipv6_am((const struct sockaddr_in6 *)laddr,
504			&s->ipv6.addr.dst, &s->ipv6.mask.dst);
505		fill_ipv6_am((const struct sockaddr_in6 *)raddr,
506			&s->ipv6.addr.src, &s->ipv6.mask.src);
507	}
508
509	rte_spinlock_lock(&ctx->dev_lock);
510	rc = stream_fill_dev(ctx, s, laddr);
511	rte_spinlock_unlock(&ctx->dev_lock);
512
513	return rc;
514}
515
516/* free stream's destination port */
517int
518stream_clear_ctx(struct tle_ctx *ctx, struct tle_stream *s)
519{
520	int32_t rc;
521
522	rte_spinlock_lock(&ctx->dev_lock);
523	rc = stream_clear_dev(ctx, s);
524	rte_spinlock_unlock(&ctx->dev_lock);
525
526	return rc;
527}
528