1/*
2 * Copyright (c) 2016-2017  Intel Corporation.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 *     http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#include <string.h>
17#include <rte_malloc.h>
18#include <rte_errno.h>
19#include <rte_cycles.h>
20#include <rte_ethdev.h>
21#include <rte_ip.h>
22
23#include "stream.h"
24#include "misc.h"
25#include <halfsiphash.h>
26
27#define	LPORT_START	0x8000
28#define	LPORT_END	MAX_PORT_NUM
29
30#define	LPORT_START_BLK	PORT_BLK(LPORT_START)
31#define	LPORT_END_BLK	PORT_BLK(LPORT_END)
32
33const struct in6_addr tle_ipv6_any = IN6ADDR_ANY_INIT;
34const struct in6_addr tle_ipv6_none = {
35	{
36		.__u6_addr32 = {
37			UINT32_MAX, UINT32_MAX, UINT32_MAX, UINT32_MAX
38		},
39	},
40};
41
42struct stream_ops tle_stream_ops[TLE_PROTO_NUM] = {};
43
44static int
45check_dev_prm(const struct tle_dev_param *dev_prm)
46{
47	/* no valid IPv4/IPv6 addresses provided. */
48	if (dev_prm->local_addr4.s_addr == INADDR_ANY &&
49			memcmp(&dev_prm->local_addr6, &tle_ipv6_any,
50			sizeof(tle_ipv6_any)) == 0)
51		return -EINVAL;
52
53	if (dev_prm->bl4.nb_port > UINT16_MAX ||
54			(dev_prm->bl4.nb_port != 0 &&
55			dev_prm->bl4.port == NULL))
56		return -EINVAL;
57
58	if (dev_prm->bl6.nb_port > UINT16_MAX ||
59			(dev_prm->bl6.nb_port != 0 &&
60			dev_prm->bl6.port == NULL))
61		return -EINVAL;
62
63	return 0;
64}
65
66static int
67check_ctx_prm(const struct tle_ctx_param *prm)
68{
69	if (prm->proto >= TLE_PROTO_NUM)
70		return -EINVAL;
71	if (prm->hash_alg >= TLE_HASH_NUM)
72		return -EINVAL;
73	return 0;
74}
75
76struct tle_ctx *
77tle_ctx_create(const struct tle_ctx_param *ctx_prm)
78{
79	struct tle_ctx *ctx;
80	size_t sz;
81	uint64_t ms;
82	uint32_t i;
83	int32_t rc;
84
85	if (ctx_prm == NULL || check_ctx_prm(ctx_prm) != 0) {
86		rte_errno = EINVAL;
87		return NULL;
88	}
89
90	sz = sizeof(*ctx);
91	ctx = rte_zmalloc_socket(NULL, sz, RTE_CACHE_LINE_SIZE,
92		ctx_prm->socket_id);
93	if (ctx == NULL) {
94		UDP_LOG(ERR, "allocation of %zu bytes for new ctx "
95			"on socket %d failed\n",
96			sz, ctx_prm->socket_id);
97		return NULL;
98	}
99
100	/* caclulate closest shift to convert from cycles to ms (approximate) */
101	ms = (rte_get_tsc_hz() + MS_PER_S - 1) / MS_PER_S;
102	ctx->cycles_ms_shift = sizeof(ms) * CHAR_BIT - __builtin_clzll(ms) - 1;
103
104	ctx->prm = *ctx_prm;
105
106	rc = tle_stream_ops[ctx_prm->proto].init_streams(ctx);
107	if (rc != 0) {
108		UDP_LOG(ERR, "init_streams(ctx=%p, proto=%u) failed "
109			"with error code: %d;\n",
110			ctx, ctx_prm->proto, rc);
111		tle_ctx_destroy(ctx);
112		rte_errno = -rc;
113		return NULL;
114	}
115
116	for (i = 0; i != RTE_DIM(ctx->use); i++)
117		tle_pbm_init(ctx->use + i, LPORT_START_BLK);
118
119	/* Initialization of siphash state is done here to speed up the
120	 * fastpath processing.
121	 */
122	if (ctx->prm.hash_alg == TLE_SIPHASH)
123		siphash_initialization(&ctx->prm.secret_key,
124					&ctx->prm.secret_key);
125	return ctx;
126}
127
128void
129tle_ctx_destroy(struct tle_ctx *ctx)
130{
131	uint32_t i;
132
133	if (ctx == NULL) {
134		rte_errno = EINVAL;
135		return;
136	}
137
138	for (i = 0; i != RTE_DIM(ctx->dev); i++)
139		tle_del_dev(ctx->dev + i);
140
141	tle_stream_ops[ctx->prm.proto].fini_streams(ctx);
142	rte_free(ctx);
143}
144
145void
146tle_ctx_invalidate(struct tle_ctx *ctx)
147{
148	RTE_SET_USED(ctx);
149}
150
151static void
152fill_pbm(struct tle_pbm *pbm, const struct tle_bl_port *blp)
153{
154	uint32_t i;
155
156	for (i = 0; i != blp->nb_port; i++)
157		tle_pbm_set(pbm, blp->port[i]);
158}
159
160static int
161init_dev_proto(struct tle_dev *dev, uint32_t idx, int32_t socket_id,
162	const struct tle_bl_port *blp)
163{
164	size_t sz;
165
166	sz = sizeof(*dev->dp[idx]);
167	dev->dp[idx] = rte_zmalloc_socket(NULL, sz, RTE_CACHE_LINE_SIZE,
168		socket_id);
169
170	if (dev->dp[idx] == NULL) {
171		UDP_LOG(ERR, "allocation of %zu bytes on "
172			"socket %d for %u-th device failed\n",
173			sz, socket_id, idx);
174		return ENOMEM;
175	}
176
177	tle_pbm_init(&dev->dp[idx]->use, LPORT_START_BLK);
178	fill_pbm(&dev->dp[idx]->use, blp);
179	return 0;
180}
181
182static struct tle_dev *
183find_free_dev(struct tle_ctx *ctx)
184{
185	uint32_t i;
186
187	if (ctx->nb_dev < RTE_DIM(ctx->dev)) {
188		for (i = 0; i != RTE_DIM(ctx->dev); i++) {
189			if (ctx->dev[i].ctx != ctx)
190				return ctx->dev + i;
191		}
192	}
193
194	rte_errno = ENODEV;
195	return NULL;
196}
197
198struct tle_dev *
199tle_add_dev(struct tle_ctx *ctx, const struct tle_dev_param *dev_prm)
200{
201	int32_t rc;
202	uint32_t df;
203	struct tle_dev *dev;
204
205	if (ctx == NULL || dev_prm == NULL || check_dev_prm(dev_prm) != 0) {
206		rte_errno = EINVAL;
207		return NULL;
208	}
209
210	dev = find_free_dev(ctx);
211	if (dev == NULL)
212		return NULL;
213	rc = 0;
214
215	/* device can handle IPv4 traffic */
216	if (dev_prm->local_addr4.s_addr != INADDR_ANY) {
217		rc = init_dev_proto(dev, TLE_V4, ctx->prm.socket_id,
218			&dev_prm->bl4);
219		if (rc == 0)
220			fill_pbm(&ctx->use[TLE_V4], &dev_prm->bl4);
221	}
222
223	/* device can handle IPv6 traffic */
224	if (rc == 0 && memcmp(&dev_prm->local_addr6, &tle_ipv6_any,
225			sizeof(tle_ipv6_any)) != 0) {
226		rc = init_dev_proto(dev, TLE_V6, ctx->prm.socket_id,
227			&dev_prm->bl6);
228		if (rc == 0)
229			fill_pbm(&ctx->use[TLE_V6], &dev_prm->bl6);
230	}
231
232	if (rc != 0) {
233		/* cleanup and return an error. */
234		rte_free(dev->dp[TLE_V4]);
235		rte_free(dev->dp[TLE_V6]);
236		rte_errno = rc;
237		return NULL;
238	}
239
240	/* setup TX data. */
241	df = ((ctx->prm.flags & TLE_CTX_FLAG_ST) == 0) ? 0 :
242		RING_F_SP_ENQ | RING_F_SC_DEQ;
243	tle_dring_reset(&dev->tx.dr, df);
244
245	if ((dev_prm->tx_offload & DEV_TX_OFFLOAD_UDP_CKSUM) != 0 &&
246			ctx->prm.proto == TLE_PROTO_UDP) {
247		dev->tx.ol_flags[TLE_V4] |= PKT_TX_IPV4 | PKT_TX_UDP_CKSUM;
248		dev->tx.ol_flags[TLE_V6] |= PKT_TX_IPV6 | PKT_TX_UDP_CKSUM;
249	} else if ((dev_prm->tx_offload & DEV_TX_OFFLOAD_TCP_CKSUM) != 0 &&
250			ctx->prm.proto == TLE_PROTO_TCP) {
251		dev->tx.ol_flags[TLE_V4] |= PKT_TX_IPV4 | PKT_TX_TCP_CKSUM;
252		dev->tx.ol_flags[TLE_V6] |= PKT_TX_IPV6 | PKT_TX_TCP_CKSUM;
253	}
254
255	if ((dev_prm->tx_offload & DEV_TX_OFFLOAD_IPV4_CKSUM) != 0)
256		dev->tx.ol_flags[TLE_V4] |= PKT_TX_IPV4 | PKT_TX_IP_CKSUM;
257
258	dev->prm = *dev_prm;
259	dev->ctx = ctx;
260	ctx->nb_dev++;
261
262	return dev;
263}
264
265static void
266empty_dring(struct tle_dring *dr, uint32_t proto)
267{
268	uint32_t i, k, n;
269	struct tle_stream *s;
270	struct rte_mbuf *pkt[MAX_PKT_BURST];
271	struct tle_drb *drb[MAX_PKT_BURST];
272
273	do {
274		k = RTE_DIM(drb);
275		n = tle_dring_sc_dequeue(dr, (const void **)(uintptr_t)pkt,
276			RTE_DIM(pkt), drb, &k);
277
278		/* free mbufs */
279		for (i = 0; i != n; i++)
280			rte_pktmbuf_free(pkt[i]);
281		/* free drbs */
282		for (i = 0; i != k; i++) {
283			s = drb[i]->udata;
284			tle_stream_ops[proto].free_drbs(s, drb + i, 1);
285		}
286	} while (n != 0);
287}
288
289int
290tle_del_dev(struct tle_dev *dev)
291{
292	uint32_t p;
293	struct tle_ctx *ctx;
294
295	if (dev == NULL || dev->ctx == NULL)
296		return -EINVAL;
297
298	ctx = dev->ctx;
299	p = dev - ctx->dev;
300
301	if (p >= RTE_DIM(ctx->dev) ||
302			(dev->dp[TLE_V4] == NULL &&
303			dev->dp[TLE_V6] == NULL))
304		return -EINVAL;
305
306	/* emtpy TX queues. */
307	empty_dring(&dev->tx.dr, ctx->prm.proto);
308
309	rte_free(dev->dp[TLE_V4]);
310	rte_free(dev->dp[TLE_V6]);
311	memset(dev, 0, sizeof(*dev));
312	ctx->nb_dev--;
313	return 0;
314}
315
316static struct tle_dev *
317find_ipv4_dev(struct tle_ctx *ctx, const struct in_addr *addr)
318{
319	uint32_t i;
320
321	for (i = 0; i != RTE_DIM(ctx->dev); i++) {
322		if (ctx->dev[i].prm.local_addr4.s_addr == addr->s_addr &&
323				ctx->dev[i].dp[TLE_V4] != NULL)
324			return ctx->dev + i;
325	}
326
327	return NULL;
328}
329
330static struct tle_dev *
331find_ipv6_dev(struct tle_ctx *ctx, const struct in6_addr *addr)
332{
333	uint32_t i;
334
335	for (i = 0; i != RTE_DIM(ctx->dev); i++) {
336		if (memcmp(&ctx->dev[i].prm.local_addr6, addr,
337				sizeof(*addr)) == 0 &&
338				ctx->dev[i].dp[TLE_V6] != NULL)
339			return ctx->dev + i;
340	}
341
342	return NULL;
343}
344
345static int
346stream_fill_dev(struct tle_ctx *ctx, struct tle_stream *s,
347	const struct sockaddr *addr)
348{
349	struct tle_dev *dev;
350	struct tle_pbm *pbm;
351	const struct sockaddr_in *lin4;
352	const struct sockaddr_in6 *lin6;
353	uint32_t i, p, sp, t;
354
355	if (addr->sa_family == AF_INET) {
356		lin4 = (const struct sockaddr_in *)addr;
357		t = TLE_V4;
358		p = lin4->sin_port;
359	} else if (addr->sa_family == AF_INET6) {
360		lin6 = (const struct sockaddr_in6 *)addr;
361		t = TLE_V6;
362		p = lin6->sin6_port;
363	} else
364		return EINVAL;
365
366	p = ntohs(p);
367
368	/* if local address is not wildcard, find device it belongs to. */
369	if (t == TLE_V4 && lin4->sin_addr.s_addr != INADDR_ANY) {
370		dev = find_ipv4_dev(ctx, &lin4->sin_addr);
371		if (dev == NULL)
372			return ENODEV;
373	} else if (t == TLE_V6 && memcmp(&tle_ipv6_any, &lin6->sin6_addr,
374			sizeof(tle_ipv6_any)) != 0) {
375		dev = find_ipv6_dev(ctx, &lin6->sin6_addr);
376		if (dev == NULL)
377			return ENODEV;
378	} else
379		dev = NULL;
380
381	if (dev != NULL)
382		pbm = &dev->dp[t]->use;
383	else
384		pbm = &ctx->use[t];
385
386	/* try to acquire local port number. */
387	if (p == 0) {
388		p = tle_pbm_find_range(pbm, pbm->blk, LPORT_END_BLK);
389		if (p == 0 && pbm->blk > LPORT_START_BLK)
390			p = tle_pbm_find_range(pbm, LPORT_START_BLK, pbm->blk);
391	} else if (tle_pbm_check(pbm, p) != 0)
392		return EEXIST;
393
394	if (p == 0)
395		return ENFILE;
396
397	/* fill socket's dst port and type */
398
399	sp = htons(p);
400	s->type = t;
401	s->port.dst = sp;
402
403	/* mark port as in-use */
404
405	tle_pbm_set(&ctx->use[t], p);
406	if (dev != NULL) {
407		tle_pbm_set(pbm, p);
408		dev->dp[t]->streams[sp] = s;
409	} else {
410		for (i = 0; i != RTE_DIM(ctx->dev); i++) {
411			if (ctx->dev[i].dp[t] != NULL) {
412				tle_pbm_set(&ctx->dev[i].dp[t]->use, p);
413				ctx->dev[i].dp[t]->streams[sp] = s;
414			}
415		}
416	}
417
418	return 0;
419}
420
421static int
422stream_clear_dev(struct tle_ctx *ctx, const struct tle_stream *s)
423{
424	struct tle_dev *dev;
425	uint32_t i, p, sp, t;
426
427	t = s->type;
428	sp = s->port.dst;
429	p = ntohs(sp);
430
431	/* if local address is not wildcard, find device it belongs to. */
432	if (t == TLE_V4 && s->ipv4.addr.dst != INADDR_ANY) {
433		dev = find_ipv4_dev(ctx,
434			(const struct in_addr *)&s->ipv4.addr.dst);
435		if (dev == NULL)
436			return ENODEV;
437	} else if (t == TLE_V6 && memcmp(&tle_ipv6_any, &s->ipv6.addr.dst,
438			sizeof(tle_ipv6_any)) != 0) {
439		dev = find_ipv6_dev(ctx,
440			(const struct in6_addr *)&s->ipv6.addr.dst);
441		if (dev == NULL)
442			return ENODEV;
443	} else
444		dev = NULL;
445
446	tle_pbm_clear(&ctx->use[t], p);
447	if (dev != NULL) {
448		if (dev->dp[t]->streams[sp] == s) {
449			tle_pbm_clear(&dev->dp[t]->use, p);
450			dev->dp[t]->streams[sp] = NULL;
451		}
452	} else {
453		for (i = 0; i != RTE_DIM(ctx->dev); i++) {
454			if (ctx->dev[i].dp[t] != NULL &&
455					ctx->dev[i].dp[t]->streams[sp] == s) {
456				tle_pbm_clear(&ctx->dev[i].dp[t]->use, p);
457				ctx->dev[i].dp[t]->streams[sp] = NULL;
458			}
459		}
460	}
461
462	return 0;
463}
464
465static void
466fill_ipv4_am(const struct sockaddr_in *in, uint32_t *addr, uint32_t *mask)
467{
468	*addr = in->sin_addr.s_addr;
469	*mask = (*addr == INADDR_ANY) ? INADDR_ANY : INADDR_NONE;
470}
471
472static void
473fill_ipv6_am(const struct sockaddr_in6 *in, rte_xmm_t *addr, rte_xmm_t *mask)
474{
475	const struct in6_addr *pm;
476
477	memcpy(addr, &in->sin6_addr, sizeof(*addr));
478	if (memcmp(&tle_ipv6_any, addr, sizeof(*addr)) == 0)
479		pm = &tle_ipv6_any;
480	else
481		pm = &tle_ipv6_none;
482
483	memcpy(mask, pm, sizeof(*mask));
484}
485
486int
487stream_fill_ctx(struct tle_ctx *ctx, struct tle_stream *s,
488	const struct sockaddr *laddr, const struct sockaddr *raddr)
489{
490	const struct sockaddr_in *rin;
491	int32_t rc;
492
493	/* setup ports and port mask fields (except dst port). */
494	rin = (const struct sockaddr_in *)raddr;
495	s->port.src = rin->sin_port;
496	s->pmsk.src = (s->port.src == 0) ? 0 : UINT16_MAX;
497	s->pmsk.dst = UINT16_MAX;
498
499	/* setup src and dst addresses. */
500	if (laddr->sa_family == AF_INET) {
501		fill_ipv4_am((const struct sockaddr_in *)laddr,
502			&s->ipv4.addr.dst, &s->ipv4.mask.dst);
503		fill_ipv4_am((const struct sockaddr_in *)raddr,
504			&s->ipv4.addr.src, &s->ipv4.mask.src);
505	} else if (laddr->sa_family == AF_INET6) {
506		fill_ipv6_am((const struct sockaddr_in6 *)laddr,
507			&s->ipv6.addr.dst, &s->ipv6.mask.dst);
508		fill_ipv6_am((const struct sockaddr_in6 *)raddr,
509			&s->ipv6.addr.src, &s->ipv6.mask.src);
510	}
511
512	rte_spinlock_lock(&ctx->dev_lock);
513	rc = stream_fill_dev(ctx, s, laddr);
514	rte_spinlock_unlock(&ctx->dev_lock);
515
516	return rc;
517}
518
519/* free stream's destination port */
520int
521stream_clear_ctx(struct tle_ctx *ctx, struct tle_stream *s)
522{
523	int32_t rc;
524
525	rte_spinlock_lock(&ctx->dev_lock);
526	rc = stream_clear_dev(ctx, s);
527	rte_spinlock_unlock(&ctx->dev_lock);
528
529	return rc;
530}
531