lcore.h revision 5c795f7b
1/*
2 * Copyright (c) 2016  Intel Corporation.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 *     http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#ifndef LCORE_H_
17#define LCORE_H_
18
19#include <rte_random.h>
20
21#include "dpdk_legacy.h"
22
23/*
24 * IPv4 destination lookup callback.
25 */
26static int
27lpm4_dst_lookup(void *data, const struct in_addr *addr,
28	struct tle_dest *res)
29{
30	int32_t rc;
31	uint32_t idx;
32	struct netbe_lcore *lc;
33	struct tle_dest *dst;
34
35	lc = data;
36
37	rc = rte_lpm_lookup(lc->lpm4, rte_be_to_cpu_32(addr->s_addr), &idx);
38	if (rc == 0) {
39		dst = &lc->dst4[idx];
40		rte_memcpy(res, dst, dst->l2_len + dst->l3_len +
41			offsetof(struct tle_dest, hdr));
42	}
43	return rc;
44}
45
46/*
47 * IPv6 destination lookup callback.
48 */
49static int
50lpm6_dst_lookup(void *data, const struct in6_addr *addr,
51	struct tle_dest *res)
52{
53	int32_t rc;
54	dpdk_lpm6_idx_t idx;
55	struct netbe_lcore *lc;
56	struct tle_dest *dst;
57	uintptr_t p;
58
59	lc = data;
60	p = (uintptr_t)addr->s6_addr;
61
62	rc = rte_lpm6_lookup(lc->lpm6, (uint8_t *)p, &idx);
63	if (rc == 0) {
64		dst = &lc->dst6[idx];
65		rte_memcpy(res, dst, dst->l2_len + dst->l3_len +
66			offsetof(struct tle_dest, hdr));
67	}
68	return rc;
69}
70
71static int
72lcore_lpm_init(struct netbe_lcore *lc)
73{
74	int32_t sid;
75	char str[RTE_LPM_NAMESIZE];
76	const struct rte_lpm_config lpm4_cfg = {
77		.max_rules = MAX_RULES,
78		.number_tbl8s = MAX_TBL8,
79	};
80	const struct rte_lpm6_config lpm6_cfg = {
81		.max_rules = MAX_RULES,
82		.number_tbl8s = MAX_TBL8,
83	};
84
85	sid = rte_lcore_to_socket_id(lc->id);
86
87	snprintf(str, sizeof(str), "LPM4%u\n", lc->id);
88	lc->lpm4 = rte_lpm_create(str, sid, &lpm4_cfg);
89	RTE_LOG(NOTICE, USER1, "%s(lcore=%u): lpm4=%p;\n",
90		__func__, lc->id, lc->lpm4);
91	if (lc->lpm4 == NULL)
92		return -ENOMEM;
93
94	snprintf(str, sizeof(str), "LPM6%u\n", lc->id);
95	lc->lpm6 = rte_lpm6_create(str, sid, &lpm6_cfg);
96	RTE_LOG(NOTICE, USER1, "%s(lcore=%u): lpm6=%p;\n",
97		__func__, lc->id, lc->lpm6);
98	if (lc->lpm6 == NULL)
99		return -ENOMEM;
100
101	return 0;
102}
103
104/*
105 * Helper functions, finds BE by given local and remote addresses.
106 */
107static int
108netbe_find4(const struct in_addr *laddr, const uint16_t lport,
109	const struct in_addr *raddr, const uint32_t belc)
110{
111	uint32_t i, j;
112	uint32_t idx;
113	struct netbe_lcore *bc;
114
115	/* we have exactly one BE, use it for all traffic */
116	if (becfg.cpu_num == 1)
117		return 0;
118
119	/* search by provided be_lcore */
120	if (belc != LCORE_ID_ANY) {
121		for (i = 0; i != becfg.cpu_num; i++) {
122			bc = becfg.cpu + i;
123			if (belc == bc->id)
124				return i;
125		}
126		RTE_LOG(NOTICE, USER1, "%s: no stream with be_lcore=%u\n",
127			__func__, belc);
128		return -ENOENT;
129	}
130
131	/* search by local address */
132	if (laddr->s_addr != INADDR_ANY) {
133		for (i = 0; i != becfg.cpu_num; i++) {
134			bc = becfg.cpu + i;
135			/* search by queue for the local port */
136			for (j = 0; j != bc->prtq_num; j++) {
137				if (laddr->s_addr == bc->prtq[j].port.ipv4) {
138
139					if (lport == 0)
140						return i;
141
142					if (verify_queue_for_port(bc->prtq + j,
143							lport) != 0)
144						return i;
145				}
146			}
147		}
148	}
149
150	/* search by remote address */
151	if (raddr->s_addr != INADDR_ANY) {
152		for (i = 0; i != becfg.cpu_num; i++) {
153			bc = becfg.cpu + i;
154			if (rte_lpm_lookup(bc->lpm4,
155					rte_be_to_cpu_32(raddr->s_addr),
156					&idx) == 0) {
157
158				if (lport == 0)
159					return i;
160
161				/* search by queue for the local port */
162				for (j = 0; j != bc->prtq_num; j++)
163					if (verify_queue_for_port(bc->prtq + j,
164							lport) != 0)
165						return i;
166			}
167		}
168	}
169
170	return -ENOENT;
171}
172
173static int
174netbe_find6(const struct in6_addr *laddr, uint16_t lport,
175	const struct in6_addr *raddr, uint32_t belc)
176{
177	uint32_t i, j;
178	dpdk_lpm6_idx_t idx;
179	struct netbe_lcore *bc;
180
181	/* we have exactly one BE, use it for all traffic */
182	if (becfg.cpu_num == 1)
183		return 0;
184
185	/* search by provided be_lcore */
186	if (belc != LCORE_ID_ANY) {
187		for (i = 0; i != becfg.cpu_num; i++) {
188			bc = becfg.cpu + i;
189			if (belc == bc->id)
190				return i;
191		}
192		RTE_LOG(NOTICE, USER1, "%s: no stream with belcore=%u\n",
193			__func__, belc);
194		return -ENOENT;
195	}
196
197	/* search by local address */
198	if (memcmp(laddr, &in6addr_any, sizeof(*laddr)) != 0) {
199		for (i = 0; i != becfg.cpu_num; i++) {
200			bc = becfg.cpu + i;
201			/* search by queue for the local port */
202			for (j = 0; j != bc->prtq_num; j++) {
203				if (memcmp(laddr, &bc->prtq[j].port.ipv6,
204						sizeof(*laddr)) == 0) {
205
206					if (lport == 0)
207						return i;
208
209					if (verify_queue_for_port(bc->prtq + j,
210							lport) != 0)
211						return i;
212				}
213			}
214		}
215	}
216
217	/* search by remote address */
218	if (memcmp(raddr, &in6addr_any, sizeof(*raddr)) == 0) {
219		for (i = 0; i != becfg.cpu_num; i++) {
220			bc = becfg.cpu + i;
221			if (rte_lpm6_lookup(bc->lpm6,
222					(uint8_t *)(uintptr_t)raddr->s6_addr,
223					&idx) == 0) {
224
225				if (lport == 0)
226					return i;
227
228				/* search by queue for the local port */
229				for (j = 0; j != bc->prtq_num; j++)
230					if (verify_queue_for_port(bc->prtq + j,
231							lport) != 0)
232						return i;
233			}
234		}
235	}
236
237	return -ENOENT;
238}
239
240static int
241create_context(struct netbe_lcore *lc, const struct tle_ctx_param *ctx_prm)
242{
243	uint32_t rc = 0, sid;
244	uint64_t frag_cycles;
245	struct tle_ctx_param cprm;
246
247	if (lc->ctx == NULL) {
248		sid = rte_lcore_to_socket_id(lc->id);
249
250		rc = lcore_lpm_init(lc);
251		if (rc != 0)
252			return rc;
253
254		cprm = *ctx_prm;
255		cprm.socket_id = sid;
256		cprm.proto = lc->proto;
257		cprm.lookup4 = lpm4_dst_lookup;
258		cprm.lookup4_data = lc;
259		cprm.lookup6 = lpm6_dst_lookup;
260		cprm.lookup6_data = lc;
261		if (cprm.secret_key.u64[0] == 0 &&
262			cprm.secret_key.u64[1] == 0) {
263			cprm.secret_key.u64[0] = rte_rand();
264			cprm.secret_key.u64[1] = rte_rand();
265		}
266
267		frag_cycles = (rte_get_tsc_hz() + MS_PER_S - 1) /
268						MS_PER_S * FRAG_TTL;
269
270		lc->ftbl = rte_ip_frag_table_create(cprm.max_streams,
271			FRAG_TBL_BUCKET_ENTRIES, cprm.max_streams,
272			frag_cycles, sid);
273
274		RTE_LOG(NOTICE, USER1, "%s(lcore=%u): frag_tbl=%p;\n",
275			__func__, lc->id, lc->ftbl);
276
277		lc->ctx = tle_ctx_create(&cprm);
278
279		RTE_LOG(NOTICE, USER1, "%s(lcore=%u): proto=%s, ctx=%p;\n",
280			__func__, lc->id, proto_name[lc->proto], lc->ctx);
281
282		if (lc->ctx == NULL || lc->ftbl == NULL)
283			rc = ENOMEM;
284	}
285
286	return rc;
287}
288
289/*
290 * BE lcore setup routine.
291 */
292static int
293lcore_init(struct netbe_lcore *lc, const struct tle_ctx_param *ctx_prm,
294	const uint32_t prtqid, const uint16_t *bl_ports, uint32_t nb_bl_ports)
295{
296	int32_t rc = 0;
297	struct tle_dev_param dprm;
298
299	rc = create_context(lc, ctx_prm);
300
301	if (rc == 0 && lc->ctx != NULL) {
302		memset(&dprm, 0, sizeof(dprm));
303		dprm.rx_offload = lc->prtq[prtqid].port.rx_offload;
304		dprm.tx_offload = lc->prtq[prtqid].port.tx_offload;
305		dprm.local_addr4.s_addr = lc->prtq[prtqid].port.ipv4;
306		memcpy(&dprm.local_addr6,  &lc->prtq[prtqid].port.ipv6,
307			sizeof(lc->prtq[prtqid].port.ipv6));
308		dprm.bl4.nb_port = nb_bl_ports;
309		dprm.bl4.port = bl_ports;
310		dprm.bl6.nb_port = nb_bl_ports;
311		dprm.bl6.port = bl_ports;
312
313		lc->prtq[prtqid].dev = tle_add_dev(lc->ctx, &dprm);
314
315		RTE_LOG(NOTICE, USER1,
316			"%s(lcore=%u, port=%u, qid=%u), dev: %p\n",
317			__func__, lc->id, lc->prtq[prtqid].port.id,
318			lc->prtq[prtqid].rxqid, lc->prtq[prtqid].dev);
319
320		if (lc->prtq[prtqid].dev == NULL)
321			rc = -rte_errno;
322
323		if (rc != 0) {
324			RTE_LOG(ERR, USER1,
325				"%s(lcore=%u) failed with error code: %d\n",
326				__func__, lc->id, rc);
327			tle_ctx_destroy(lc->ctx);
328			rte_ip_frag_table_destroy(lc->ftbl);
329			rte_lpm_free(lc->lpm4);
330			rte_lpm6_free(lc->lpm6);
331			rte_free(lc->prtq[prtqid].port.lcore_id);
332			lc->prtq[prtqid].port.nb_lcore = 0;
333			rte_free(lc->prtq);
334			lc->prtq_num = 0;
335			return rc;
336		}
337	}
338
339	return rc;
340}
341
342static uint16_t
343create_blocklist(const struct netbe_port *beprt, uint16_t *bl_ports,
344	uint32_t q)
345{
346	uint32_t i, j, qid, align_nb_q;
347
348	align_nb_q = rte_align32pow2(beprt->nb_lcore);
349	for (i = 0, j = 0; i < (UINT16_MAX + 1); i++) {
350		qid = (i % align_nb_q) % beprt->nb_lcore;
351		if (qid != q)
352			bl_ports[j++] = i;
353	}
354
355	return j;
356}
357
358static int
359netbe_lcore_init(struct netbe_cfg *cfg, const struct tle_ctx_param *ctx_prm)
360{
361	int32_t rc;
362	uint32_t i, j, nb_bl_ports = 0, sz;
363	struct netbe_lcore *lc;
364	static uint16_t *bl_ports;
365
366	/* Create the context and attached queue for each lcore. */
367	rc = 0;
368	sz = sizeof(uint16_t) * UINT16_MAX;
369	bl_ports = rte_zmalloc(NULL, sz, RTE_CACHE_LINE_SIZE);
370	for (i = 0; i < cfg->cpu_num; i++) {
371		lc = &cfg->cpu[i];
372		for (j = 0; j < lc->prtq_num; j++) {
373			memset((uint8_t *)bl_ports, 0, sz);
374			/* create list of blocked ports based on q */
375			nb_bl_ports = create_blocklist(&lc->prtq[j].port,
376				bl_ports, lc->prtq[j].rxqid);
377			RTE_LOG(NOTICE, USER1,
378				"lc=%u, q=%u, nb_bl_ports=%u\n",
379				lc->id, lc->prtq[j].rxqid, nb_bl_ports);
380
381			rc = lcore_init(lc, ctx_prm, j, bl_ports, nb_bl_ports);
382			if (rc != 0) {
383				RTE_LOG(ERR, USER1,
384					"%s: failed with error code: %d\n",
385					__func__, rc);
386				rte_free(bl_ports);
387				return rc;
388			}
389		}
390	}
391	rte_free(bl_ports);
392
393	return 0;
394}
395
396static int
397netfe_lcore_cmp(const void *s1, const void *s2)
398{
399	const struct netfe_stream_prm *p1, *p2;
400
401	p1 = s1;
402	p2 = s2;
403	return p1->lcore - p2->lcore;
404}
405
406static int
407netbe_find(const struct sockaddr_storage *la,
408	const struct sockaddr_storage *ra,
409	uint32_t belc)
410{
411	const struct sockaddr_in *l4, *r4;
412	const struct sockaddr_in6 *l6, *r6;
413
414	if (la->ss_family == AF_INET) {
415		l4 = (const struct sockaddr_in *)la;
416		r4 = (const struct sockaddr_in *)ra;
417		return netbe_find4(&l4->sin_addr, ntohs(l4->sin_port),
418				&r4->sin_addr, belc);
419	} else if (la->ss_family == AF_INET6) {
420		l6 = (const struct sockaddr_in6 *)la;
421		r6 = (const struct sockaddr_in6 *)ra;
422		return netbe_find6(&l6->sin6_addr, ntohs(l6->sin6_port),
423				&r6->sin6_addr, belc);
424	}
425	return -EINVAL;
426}
427
428static int
429netfe_sprm_flll_be(struct netfe_sprm *sp, uint32_t line, uint32_t belc)
430{
431	int32_t bidx;
432
433	bidx = netbe_find(&sp->local_addr, &sp->remote_addr, belc);
434
435	if (bidx < 0) {
436		RTE_LOG(ERR, USER1, "%s(line=%u): no BE for that stream\n",
437			__func__, line);
438		return -EINVAL;
439	}
440	sp->bidx = bidx;
441	return 0;
442}
443
444/* start front-end processing. */
445static int
446netfe_lcore_fill(struct lcore_prm prm[RTE_MAX_LCORE],
447	struct netfe_lcore_prm *lprm)
448{
449	uint32_t belc;
450	uint32_t i, j, lc, ln;
451	struct netfe_stream_prm *s;
452
453	/* determine on what BE each stream should be open. */
454	for (i = 0; i != lprm->nb_streams; i++) {
455		s = lprm->stream + i;
456		ln = s->line;
457		belc = s->belcore;
458		if (netfe_sprm_flll_be(&s->sprm, ln, belc) != 0 ||
459				(s->op == FWD &&
460				netfe_sprm_flll_be(&s->fprm, ln, belc) != 0))
461			return -EINVAL;
462	}
463
464	/* group all fe parameters by lcore. */
465
466	qsort(lprm->stream, lprm->nb_streams, sizeof(lprm->stream[0]),
467		netfe_lcore_cmp);
468
469	for (i = 0; i != lprm->nb_streams; i = j) {
470
471		lc = lprm->stream[i].lcore;
472		ln = lprm->stream[i].line;
473
474		if (rte_lcore_is_enabled(lc) == 0) {
475			RTE_LOG(ERR, USER1,
476				"%s(line=%u): lcore %u is not enabled\n",
477				__func__, ln, lc);
478			return -EINVAL;
479		}
480
481		if (rte_get_master_lcore() != lc &&
482				rte_eal_get_lcore_state(lc) == RUNNING) {
483			RTE_LOG(ERR, USER1,
484				"%s(line=%u): lcore %u already in use\n",
485				__func__, ln, lc);
486			return -EINVAL;
487		}
488
489		for (j = i + 1; j != lprm->nb_streams &&
490				lc == lprm->stream[j].lcore;
491				j++)
492			;
493
494		prm[lc].fe.max_streams = lprm->max_streams;
495		prm[lc].fe.nb_streams = j - i;
496		prm[lc].fe.stream = lprm->stream + i;
497	}
498
499	return 0;
500}
501
502#endif /* LCORE_H_ */
503