ctx.c revision 7e18fa1b
1/*
2 * Copyright (c) 2016-2017  Intel Corporation.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 *     http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#include <string.h>
17#include <rte_malloc.h>
18#include <rte_errno.h>
19#include <rte_cycles.h>
20#include <rte_ethdev.h>
21#include <rte_ip.h>
22
23#include "stream.h"
24#include "misc.h"
25#include <halfsiphash.h>
26
27#define	LPORT_START	0x8000
28#define	LPORT_END	MAX_PORT_NUM
29
30#define	LPORT_START_BLK	PORT_BLK(LPORT_START)
31#define	LPORT_END_BLK	PORT_BLK(LPORT_END)
32
33const struct in6_addr tle_ipv6_any = IN6ADDR_ANY_INIT;
34const struct in6_addr tle_ipv6_none = {
35	{
36		.__u6_addr32 = {
37			UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX
38		},
39	},
40};
41
42struct stream_ops tle_stream_ops[TLE_PROTO_NUM] = {};
43
44static int
45check_dev_prm(const struct tle_dev_param *dev_prm)
46{
47	/* no valid IPv4/IPv6 addresses provided. */
48	if (dev_prm->local_addr4.s_addr == INADDR_ANY &&
49			memcmp(&dev_prm->local_addr6, &tle_ipv6_any,
50			sizeof(tle_ipv6_any)) == 0)
51		return -EINVAL;
52
53	if (dev_prm->bl4.nb_port > UINT16_MAX ||
54			(dev_prm->bl4.nb_port != 0 &&
55			dev_prm->bl4.port == NULL))
56		return -EINVAL;
57
58	if (dev_prm->bl6.nb_port > UINT16_MAX ||
59			(dev_prm->bl6.nb_port != 0 &&
60			dev_prm->bl6.port == NULL))
61		return -EINVAL;
62
63	return 0;
64}
65
66static int
67check_ctx_prm(const struct tle_ctx_param *prm)
68{
69	if (prm->proto >= TLE_PROTO_NUM)
70		return -EINVAL;
71	if (prm->hash_alg >= TLE_HASH_NUM)
72		return -EINVAL;
73	return 0;
74}
75
76struct tle_ctx *
77tle_ctx_create(const struct tle_ctx_param *ctx_prm)
78{
79	struct tle_ctx *ctx;
80	size_t sz;
81	uint64_t ms;
82	uint32_t i;
83	int32_t rc;
84
85	if (ctx_prm == NULL || check_ctx_prm(ctx_prm) != 0) {
86		rte_errno = EINVAL;
87		return NULL;
88	}
89
90	sz = sizeof(*ctx);
91	ctx = rte_zmalloc_socket(NULL, sz, RTE_CACHE_LINE_SIZE,
92		ctx_prm->socket_id);
93	if (ctx == NULL) {
94		UDP_LOG(ERR, "allocation of %zu bytes for new ctx "
95			"on socket %d failed\n",
96			sz, ctx_prm->socket_id);
97		return NULL;
98	}
99
100	/* caclulate closest shift to convert from cycles to ms (approximate) */
101	ms = (rte_get_tsc_hz() + MS_PER_S - 1) / MS_PER_S;
102	ctx->cycles_ms_shift = sizeof(ms) * CHAR_BIT - __builtin_clzll(ms) - 1;
103
104	ctx->prm = *ctx_prm;
105
106	rc = tle_stream_ops[ctx_prm->proto].init_streams(ctx);
107	if (rc != 0) {
108		UDP_LOG(ERR, "init_streams(ctx=%p, proto=%u) failed "
109			"with error code: %d;\n",
110			ctx, ctx_prm->proto, rc);
111		tle_ctx_destroy(ctx);
112		rte_errno = -rc;
113		return NULL;
114	}
115
116	for (i = 0; i != RTE_DIM(ctx->use); i++)
117		tle_pbm_init(ctx->use + i, LPORT_START_BLK);
118
119	ctx->streams.nb_free = ctx->prm.max_streams;
120
121	/* Initialization of siphash state is done here to speed up the
122	 * fastpath processing.
123	 */
124	if (ctx->prm.hash_alg == TLE_SIPHASH)
125		siphash_initialization(&ctx->prm.secret_key,
126					&ctx->prm.secret_key);
127	return ctx;
128}
129
130void
131tle_ctx_destroy(struct tle_ctx *ctx)
132{
133	uint32_t i;
134
135	if (ctx == NULL) {
136		rte_errno = EINVAL;
137		return;
138	}
139
140	for (i = 0; i != RTE_DIM(ctx->dev); i++)
141		tle_del_dev(ctx->dev + i);
142
143	tle_stream_ops[ctx->prm.proto].fini_streams(ctx);
144	rte_free(ctx);
145}
146
147void
148tle_ctx_invalidate(struct tle_ctx *ctx)
149{
150	RTE_SET_USED(ctx);
151}
152
153static void
154fill_pbm(struct tle_pbm *pbm, const struct tle_bl_port *blp)
155{
156	uint32_t i;
157
158	for (i = 0; i != blp->nb_port; i++)
159		tle_pbm_set(pbm, blp->port[i]);
160}
161
162static int
163init_dev_proto(struct tle_dev *dev, uint32_t idx, int32_t socket_id,
164	const struct tle_bl_port *blp)
165{
166	size_t sz;
167
168	sz = sizeof(*dev->dp[idx]);
169	dev->dp[idx] = rte_zmalloc_socket(NULL, sz, RTE_CACHE_LINE_SIZE,
170		socket_id);
171
172	if (dev->dp[idx] == NULL) {
173		UDP_LOG(ERR, "allocation of %zu bytes on "
174			"socket %d for %u-th device failed\n",
175			sz, socket_id, idx);
176		return ENOMEM;
177	}
178
179	tle_pbm_init(&dev->dp[idx]->use, LPORT_START_BLK);
180	fill_pbm(&dev->dp[idx]->use, blp);
181	return 0;
182}
183
184static struct tle_dev *
185find_free_dev(struct tle_ctx *ctx)
186{
187	uint32_t i;
188
189	if (ctx->nb_dev < RTE_DIM(ctx->dev)) {
190		for (i = 0; i != RTE_DIM(ctx->dev); i++) {
191			if (ctx->dev[i].ctx != ctx)
192				return ctx->dev + i;
193		}
194	}
195
196	rte_errno = ENODEV;
197	return NULL;
198}
199
200struct tle_dev *
201tle_add_dev(struct tle_ctx *ctx, const struct tle_dev_param *dev_prm)
202{
203	int32_t rc;
204	uint32_t df;
205	struct tle_dev *dev;
206
207	if (ctx == NULL || dev_prm == NULL || check_dev_prm(dev_prm) != 0) {
208		rte_errno = EINVAL;
209		return NULL;
210	}
211
212	dev = find_free_dev(ctx);
213	if (dev == NULL)
214		return NULL;
215	rc = 0;
216
217	/* device can handle IPv4 traffic */
218	if (dev_prm->local_addr4.s_addr != INADDR_ANY) {
219		rc = init_dev_proto(dev, TLE_V4, ctx->prm.socket_id,
220			&dev_prm->bl4);
221		if (rc == 0)
222			fill_pbm(&ctx->use[TLE_V4], &dev_prm->bl4);
223	}
224
225	/* device can handle IPv6 traffic */
226	if (rc == 0 && memcmp(&dev_prm->local_addr6, &tle_ipv6_any,
227			sizeof(tle_ipv6_any)) != 0) {
228		rc = init_dev_proto(dev, TLE_V6, ctx->prm.socket_id,
229			&dev_prm->bl6);
230		if (rc == 0)
231			fill_pbm(&ctx->use[TLE_V6], &dev_prm->bl6);
232	}
233
234	if (rc != 0) {
235		/* cleanup and return an error. */
236		rte_free(dev->dp[TLE_V4]);
237		rte_free(dev->dp[TLE_V6]);
238		rte_errno = rc;
239		return NULL;
240	}
241
242	/* setup RX data. */
243	if (dev_prm->local_addr4.s_addr != INADDR_ANY &&
244			(dev_prm->rx_offload & DEV_RX_OFFLOAD_IPV4_CKSUM) == 0)
245		dev->rx.ol_flags[TLE_V4] |= PKT_RX_IP_CKSUM_BAD;
246
247	if (((dev_prm->rx_offload & DEV_RX_OFFLOAD_UDP_CKSUM) == 0 &&
248			ctx->prm.proto == TLE_PROTO_UDP) ||
249			((dev_prm->rx_offload &
250			DEV_RX_OFFLOAD_TCP_CKSUM) == 0 &&
251			ctx->prm.proto == TLE_PROTO_TCP)) {
252		dev->rx.ol_flags[TLE_V4] |= PKT_RX_L4_CKSUM_BAD;
253		dev->rx.ol_flags[TLE_V6] |= PKT_RX_L4_CKSUM_BAD;
254	}
255
256	/* setup TX data. */
257	df = ((ctx->prm.flags & TLE_CTX_FLAG_ST) == 0) ? 0 :
258		RING_F_SP_ENQ | RING_F_SC_DEQ;
259	tle_dring_reset(&dev->tx.dr, df);
260
261	if ((dev_prm->tx_offload & DEV_TX_OFFLOAD_UDP_CKSUM) != 0 &&
262			ctx->prm.proto == TLE_PROTO_UDP) {
263		dev->tx.ol_flags[TLE_V4] |= PKT_TX_IPV4 | PKT_TX_UDP_CKSUM;
264		dev->tx.ol_flags[TLE_V6] |= PKT_TX_IPV6 | PKT_TX_UDP_CKSUM;
265	} else if ((dev_prm->tx_offload & DEV_TX_OFFLOAD_TCP_CKSUM) != 0 &&
266			ctx->prm.proto == TLE_PROTO_TCP) {
267		dev->tx.ol_flags[TLE_V4] |= PKT_TX_IPV4 | PKT_TX_TCP_CKSUM;
268		dev->tx.ol_flags[TLE_V6] |= PKT_TX_IPV6 | PKT_TX_TCP_CKSUM;
269	}
270
271	if ((dev_prm->tx_offload & DEV_TX_OFFLOAD_IPV4_CKSUM) != 0)
272		dev->tx.ol_flags[TLE_V4] |= PKT_TX_IPV4 | PKT_TX_IP_CKSUM;
273
274	dev->prm = *dev_prm;
275	dev->ctx = ctx;
276	ctx->nb_dev++;
277
278	return dev;
279}
280
281static void
282empty_dring(struct tle_dring *dr, uint32_t proto)
283{
284	uint32_t i, k, n;
285	struct tle_stream *s;
286	struct rte_mbuf *pkt[MAX_PKT_BURST];
287	struct tle_drb *drb[MAX_PKT_BURST];
288
289	do {
290		k = RTE_DIM(drb);
291		n = tle_dring_sc_dequeue(dr, (const void **)(uintptr_t)pkt,
292			RTE_DIM(pkt), drb, &k);
293
294		/* free mbufs */
295		for (i = 0; i != n; i++)
296			rte_pktmbuf_free(pkt[i]);
297		/* free drbs */
298		for (i = 0; i != k; i++) {
299			s = drb[i]->udata;
300			tle_stream_ops[proto].free_drbs(s, drb + i, 1);
301		}
302	} while (n != 0);
303}
304
305int
306tle_del_dev(struct tle_dev *dev)
307{
308	uint32_t p;
309	struct tle_ctx *ctx;
310
311	if (dev == NULL || dev->ctx == NULL)
312		return -EINVAL;
313
314	ctx = dev->ctx;
315	p = dev - ctx->dev;
316
317	if (p >= RTE_DIM(ctx->dev) ||
318			(dev->dp[TLE_V4] == NULL &&
319			dev->dp[TLE_V6] == NULL))
320		return -EINVAL;
321
322	/* emtpy TX queues. */
323	empty_dring(&dev->tx.dr, ctx->prm.proto);
324
325	rte_free(dev->dp[TLE_V4]);
326	rte_free(dev->dp[TLE_V6]);
327	memset(dev, 0, sizeof(*dev));
328	ctx->nb_dev--;
329	return 0;
330}
331
332static struct tle_dev *
333find_ipv4_dev(struct tle_ctx *ctx, const struct in_addr *addr)
334{
335	uint32_t i;
336
337	for (i = 0; i != RTE_DIM(ctx->dev); i++) {
338		if (ctx->dev[i].prm.local_addr4.s_addr == addr->s_addr &&
339				ctx->dev[i].dp[TLE_V4] != NULL)
340			return ctx->dev + i;
341	}
342
343	return NULL;
344}
345
346static struct tle_dev *
347find_ipv6_dev(struct tle_ctx *ctx, const struct in6_addr *addr)
348{
349	uint32_t i;
350
351	for (i = 0; i != RTE_DIM(ctx->dev); i++) {
352		if (memcmp(&ctx->dev[i].prm.local_addr6, addr,
353				sizeof(*addr)) == 0 &&
354				ctx->dev[i].dp[TLE_V6] != NULL)
355			return ctx->dev + i;
356	}
357
358	return NULL;
359}
360
361static int
362stream_fill_dev(struct tle_ctx *ctx, struct tle_stream *s,
363	const struct sockaddr *addr)
364{
365	struct tle_dev *dev;
366	struct tle_pbm *pbm;
367	const struct sockaddr_in *lin4;
368	const struct sockaddr_in6 *lin6;
369	uint32_t i, p, sp, t;
370
371	if (addr->sa_family == AF_INET) {
372		lin4 = (const struct sockaddr_in *)addr;
373		t = TLE_V4;
374		p = lin4->sin_port;
375	} else if (addr->sa_family == AF_INET6) {
376		lin6 = (const struct sockaddr_in6 *)addr;
377		t = TLE_V6;
378		p = lin6->sin6_port;
379	} else
380		return EINVAL;
381
382	p = ntohs(p);
383
384	/* if local address is not wildcard, find device it belongs to. */
385	if (t == TLE_V4 && lin4->sin_addr.s_addr != INADDR_ANY) {
386		dev = find_ipv4_dev(ctx, &lin4->sin_addr);
387		if (dev == NULL)
388			return ENODEV;
389	} else if (t == TLE_V6 && memcmp(&tle_ipv6_any, &lin6->sin6_addr,
390			sizeof(tle_ipv6_any)) != 0) {
391		dev = find_ipv6_dev(ctx, &lin6->sin6_addr);
392		if (dev == NULL)
393			return ENODEV;
394	} else
395		dev = NULL;
396
397	if (dev != NULL)
398		pbm = &dev->dp[t]->use;
399	else
400		pbm = &ctx->use[t];
401
402	/* try to acquire local port number. */
403	if (p == 0) {
404		p = tle_pbm_find_range(pbm, pbm->blk, LPORT_END_BLK);
405		if (p == 0 && pbm->blk > LPORT_START_BLK)
406			p = tle_pbm_find_range(pbm, LPORT_START_BLK, pbm->blk);
407	} else if (tle_pbm_check(pbm, p) != 0)
408		return EEXIST;
409
410	if (p == 0)
411		return ENFILE;
412
413	/* fill socket's dst port and type */
414
415	sp = htons(p);
416	s->type = t;
417	s->port.dst = sp;
418
419	/* mark port as in-use */
420
421	tle_pbm_set(&ctx->use[t], p);
422	if (dev != NULL) {
423		tle_pbm_set(pbm, p);
424		dev->dp[t]->streams[sp] = s;
425	} else {
426		for (i = 0; i != RTE_DIM(ctx->dev); i++) {
427			if (ctx->dev[i].dp[t] != NULL) {
428				tle_pbm_set(&ctx->dev[i].dp[t]->use, p);
429				ctx->dev[i].dp[t]->streams[sp] = s;
430			}
431		}
432	}
433
434	return 0;
435}
436
437static int
438stream_clear_dev(struct tle_ctx *ctx, const struct tle_stream *s)
439{
440	struct tle_dev *dev;
441	uint32_t i, p, sp, t;
442
443	t = s->type;
444	sp = s->port.dst;
445	p = ntohs(sp);
446
447	/* if local address is not wildcard, find device it belongs to. */
448	if (t == TLE_V4 && s->ipv4.addr.dst != INADDR_ANY) {
449		dev = find_ipv4_dev(ctx,
450			(const struct in_addr *)&s->ipv4.addr.dst);
451		if (dev == NULL)
452			return ENODEV;
453	} else if (t == TLE_V6 && memcmp(&tle_ipv6_any, &s->ipv6.addr.dst,
454			sizeof(tle_ipv6_any)) != 0) {
455		dev = find_ipv6_dev(ctx,
456			(const struct in6_addr *)&s->ipv6.addr.dst);
457		if (dev == NULL)
458			return ENODEV;
459	} else
460		dev = NULL;
461
462	tle_pbm_clear(&ctx->use[t], p);
463	if (dev != NULL) {
464		if (dev->dp[t]->streams[sp] == s) {
465			tle_pbm_clear(&dev->dp[t]->use, p);
466			dev->dp[t]->streams[sp] = NULL;
467		}
468	} else {
469		for (i = 0; i != RTE_DIM(ctx->dev); i++) {
470			if (ctx->dev[i].dp[t] != NULL &&
471					ctx->dev[i].dp[t]->streams[sp] == s) {
472				tle_pbm_clear(&ctx->dev[i].dp[t]->use, p);
473				ctx->dev[i].dp[t]->streams[sp] = NULL;
474			}
475		}
476	}
477
478	return 0;
479}
480
481static void
482fill_ipv4_am(const struct sockaddr_in *in, uint32_t *addr, uint32_t *mask)
483{
484	*addr = in->sin_addr.s_addr;
485	*mask = (*addr == INADDR_ANY) ? INADDR_ANY : INADDR_NONE;
486}
487
488static void
489fill_ipv6_am(const struct sockaddr_in6 *in, rte_xmm_t *addr, rte_xmm_t *mask)
490{
491	const struct in6_addr *pm;
492
493	memcpy(addr, &in->sin6_addr, sizeof(*addr));
494	if (memcmp(&tle_ipv6_any, addr, sizeof(*addr)) == 0)
495		pm = &tle_ipv6_any;
496	else
497		pm = &tle_ipv6_none;
498
499	memcpy(mask, pm, sizeof(*mask));
500}
501
502int
503stream_fill_ctx(struct tle_ctx *ctx, struct tle_stream *s,
504	const struct sockaddr *laddr, const struct sockaddr *raddr)
505{
506	const struct sockaddr_in *rin;
507	int32_t rc;
508
509	/* setup ports and port mask fields (except dst port). */
510	rin = (const struct sockaddr_in *)raddr;
511	s->port.src = rin->sin_port;
512	s->pmsk.src = (s->port.src == 0) ? 0 : UINT16_MAX;
513	s->pmsk.dst = UINT16_MAX;
514
515	/* setup src and dst addresses. */
516	if (laddr->sa_family == AF_INET) {
517		fill_ipv4_am((const struct sockaddr_in *)laddr,
518			&s->ipv4.addr.dst, &s->ipv4.mask.dst);
519		fill_ipv4_am((const struct sockaddr_in *)raddr,
520			&s->ipv4.addr.src, &s->ipv4.mask.src);
521	} else if (laddr->sa_family == AF_INET6) {
522		fill_ipv6_am((const struct sockaddr_in6 *)laddr,
523			&s->ipv6.addr.dst, &s->ipv6.mask.dst);
524		fill_ipv6_am((const struct sockaddr_in6 *)raddr,
525			&s->ipv6.addr.src, &s->ipv6.mask.src);
526	}
527
528	rte_spinlock_lock(&ctx->dev_lock);
529	rc = stream_fill_dev(ctx, s, laddr);
530	rte_spinlock_unlock(&ctx->dev_lock);
531
532	return rc;
533}
534
535/* free stream's destination port */
536int
537stream_clear_ctx(struct tle_ctx *ctx, struct tle_stream *s)
538{
539	int32_t rc;
540
541	rte_spinlock_lock(&ctx->dev_lock);
542	rc = stream_clear_dev(ctx, s);
543	rte_spinlock_unlock(&ctx->dev_lock);
544
545	return rc;
546}
547