be.c revision 5c795f7b
1/*
2 * Copyright (c) 2017  Intel Corporation.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 *    notice, this list of conditions and the following disclaimer in the
12 *    documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27#include <assert.h>
28#include <netinet/ip6.h>
29
30#include <ngx_config.h>
31#include <ngx_core.h>
32
33#include "be.h"
34#include <rte_version.h>
35#include <rte_cycles.h>
36#include <rte_ethdev.h>
37#include <rte_errno.h>
38#include <rte_lpm6.h>
39#include <rte_lpm.h>
40#include <rte_ip.h>
41#include <rte_tcp.h>
42
43#include <tle_tcp.h>
44
45#if RTE_VERSION_NUM(17, 11, 0, 0) <= RTE_VERSION
46typedef uint16_t dpdk_port_t;
47#else
48typedef uint8_t dpdk_port_t;
49#endif
50
51#define RX_RING_SIZE    0x400
52#define TX_RING_SIZE    0x800
53#define MAX_RULES       0x100
54#define MAX_TBL8        0x800
55
56#define MPOOL_CACHE_SIZE        0x100
57#define MPOOL_NB_BUF            0x20000
58
59#define FRAG_MBUF_BUF_SIZE      (RTE_PKTMBUF_HEADROOM + TLE_DST_MAX_HDR)
60
61#define RX_CSUM_OFFLOAD (DEV_RX_OFFLOAD_IPV4_CKSUM | DEV_RX_OFFLOAD_TCP_CKSUM)
62
63#define TCP_MAX_PROCESS 0x20
64
65static const struct rte_eth_conf port_conf_default = {
66	.rxmode = {
67		.hw_vlan_strip = 1,
68	},
69};
70
71struct ptype2cb {
72	uint32_t mask;
73	const char *name;
74	rte_rx_callback_fn fn;
75};
76
77enum {
78	ETHER_PTYPE = 0x1,
79	IPV4_PTYPE = 0x2,
80	IPV4_EXT_PTYPE = 0x4,
81	IPV6_PTYPE = 0x8,
82	IPV6_EXT_PTYPE = 0x10,
83	TCP_PTYPE = 0x20,
84	UDP_PTYPE = 0x40,
85};
86
87int
88be_lcore_lpm_init(struct tldk_ctx *tcx, uint32_t sid,
89		const struct tldk_ctx_conf *cf)
90{
91	ngx_uint_t worker = cf->worker;
92	uint32_t lcore = cf->lcore;
93	char str[RTE_LPM_NAMESIZE];
94
95	const struct rte_lpm_config lpm4_cfg = {
96		.max_rules = MAX_RULES,
97		.number_tbl8s = MAX_TBL8,
98	};
99
100	const struct rte_lpm6_config lpm6_cfg = {
101		.max_rules = MAX_RULES,
102		.number_tbl8s = MAX_TBL8,
103	};
104
105	snprintf(str, sizeof(str), "LPM4%lu-%u\n", worker, lcore);
106	tcx->lpm4 = rte_lpm_create(str, sid, &lpm4_cfg);
107	RTE_LOG(NOTICE, USER1, "%s(worker=%lu, lcore=%u): lpm4=%p;\n",
108		__func__, worker, lcore, tcx->lpm4);
109	if (tcx->lpm4 == NULL)
110		return -ENOMEM;
111
112	snprintf(str, sizeof(str), "LPM6%lu-%u\n", worker, lcore);
113	tcx->lpm6 = rte_lpm6_create(str, sid, &lpm6_cfg);
114	RTE_LOG(NOTICE, USER1, "%s(worker=%lu, lcore=%u): lpm6=%p;\n",
115		__func__, worker, lcore, tcx->lpm6);
116	if (tcx->lpm6 == NULL) {
117		rte_lpm_free(tcx->lpm4);
118		return -ENOMEM;
119	}
120
121	return 0;
122}
123
124int
125be_lpm4_dst_lookup(void *data, const struct in_addr *addr,
126		struct tle_dest *res)
127{
128	int32_t rc;
129	uint32_t idx;
130	struct tldk_ctx *tcx;
131	struct tle_dest *dst;
132
133	tcx = data;
134	rc = rte_lpm_lookup(tcx->lpm4, rte_be_to_cpu_32(addr->s_addr), &idx);
135	if (rc == 0) {
136		dst = &tcx->dst4[idx];
137		memcpy(res, dst, dst->l2_len + dst->l3_len +
138				offsetof(struct tle_dest, hdr));
139	}
140
141	return rc;
142}
143
144int
145be_lpm6_dst_lookup(void *data, const struct in6_addr *addr,
146	struct tle_dest *res)
147{
148	int32_t rc;
149	struct tldk_ctx *tcx;
150	struct tle_dest *dst;
151	uintptr_t p;
152#if RTE_VERSION_NUM(17, 5, 0, 0) <= RTE_VERSION
153	uint32_t idx;
154#else
155	uint8_t idx;
156#endif
157
158	tcx = data;
159	p = (uintptr_t)addr->s6_addr;
160	rc = rte_lpm6_lookup(tcx->lpm6, (uint8_t *)p, &idx);
161	if (rc == 0) {
162		dst = &tcx->dst6[idx];
163		memcpy(res, dst, dst->l2_len + dst->l3_len +
164				offsetof(struct tle_dest, hdr));
165	}
166
167	return rc;
168}
169
170/*
171 * Initialise DPDK port.
172 */
173static int
174port_init(const struct tldk_port_conf *pcf)
175{
176	int32_t rc;
177	struct rte_eth_conf port_conf;
178	struct rte_eth_dev_info dev_info;
179
180	rte_eth_dev_info_get(pcf->id, &dev_info);
181
182	if ((dev_info.rx_offload_capa & pcf->rx_offload) != pcf->rx_offload) {
183		RTE_LOG(ERR, USER1,
184			"port#%u supported/requested RX offloads don't match, "
185			"supported: %#" PRIx64 ", requested: %#" PRIx64 ";\n",
186			pcf->id, (uint64_t)dev_info.rx_offload_capa,
187			pcf->rx_offload);
188		return NGX_ERROR;
189	}
190	if ((dev_info.tx_offload_capa & pcf->tx_offload) != pcf->tx_offload) {
191		RTE_LOG(ERR, USER1,
192			"port#%u supported/requested TX offloads don't match, "
193			"supported: %#" PRIx64 ", requested: %#" PRIx64 ";\n",
194			pcf->id, (uint64_t)dev_info.tx_offload_capa,
195			pcf->tx_offload);
196		return NGX_ERROR;
197	}
198
199	port_conf = port_conf_default;
200
201	if ((pcf->rx_offload & RX_CSUM_OFFLOAD) != 0) {
202		RTE_LOG(ERR, USER1, "%s(%u): enabling RX csum offload;\n",
203			__func__, pcf->id);
204		port_conf.rxmode.hw_ip_checksum = 1;
205	}
206
207	port_conf.rxmode.max_rx_pkt_len = pcf->mtu + ETHER_CRC_LEN;
208	if (port_conf.rxmode.max_rx_pkt_len > ETHER_MAX_LEN)
209		port_conf.rxmode.jumbo_frame = 1;
210	port_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
211	port_conf.rx_adv_conf.rss_conf.rss_hf = ETH_RSS_IP | ETH_RSS_TCP;
212
213	rc = rte_eth_dev_configure(pcf->id, pcf->nb_queues, pcf->nb_queues,
214			&port_conf);
215	RTE_LOG(NOTICE, USER1,
216		"%s: rte_eth_dev_configure(prt_id=%u, nb_rxq=%u, nb_txq=%u) "
217		"returns %d;\n", __func__, pcf->id, pcf->nb_queues,
218		pcf->nb_queues, rc);
219
220	if (rc != 0)
221		return NGX_ERROR;
222
223	return NGX_OK;
224}
225
226/*
227 * Check that lcore is enabled, not master, and not in use already.
228 */
229int
230be_check_lcore(uint32_t lid)
231{
232	if (rte_lcore_is_enabled(lid) == 0) {
233		RTE_LOG(ERR, USER1, "lcore %u is not enabled\n", lid);
234		return -EINVAL;
235	}
236
237	if (rte_get_master_lcore() != lid &&
238		rte_eal_get_lcore_state(lid) == RUNNING) {
239		RTE_LOG(ERR, USER1, "lcore %u already running %p\n",
240			lid, lcore_config[lid].f);
241		return -EINVAL;
242	}
243
244	return 0;
245}
246
247int
248be_mpool_init(struct tldk_ctx *tcx)
249{
250	int32_t rc;
251	uint32_t nmb, sid;
252	struct rte_mempool *mp;
253	char name[RTE_MEMPOOL_NAMESIZE];
254
255	ngx_uint_t worker = tcx->cf->worker;
256	uint32_t lcore = tcx->cf->lcore;
257
258	sid = rte_lcore_to_socket_id(tcx->cf->lcore);
259	nmb = (tcx->cf->nb_mbuf == 0) ? MPOOL_NB_BUF : tcx->cf->nb_mbuf;
260
261	snprintf(name, sizeof(name), "MP%lu-%u", worker, lcore);
262	mp = rte_pktmbuf_pool_create(name, nmb, MPOOL_CACHE_SIZE, 0,
263			RTE_MBUF_DEFAULT_BUF_SIZE, sid);
264	if (mp == NULL) {
265		rc = -rte_errno;
266		RTE_LOG(ERR, USER1, "%s:Mempool creation failed for "
267			"ctx:wrk(%lu)-ctx:lcore(%u) with error code: %d\n",
268			__func__, worker, lcore, rc);
269		return rc;
270	}
271
272	tcx->mpool = mp;
273
274	snprintf(name, sizeof(name), "frag_MP%lu-%u",
275			worker, lcore);
276	mp = rte_pktmbuf_pool_create(name, nmb,
277			MPOOL_CACHE_SIZE, 0, FRAG_MBUF_BUF_SIZE, sid - 1);
278	if (mp == NULL) {
279		rc = -rte_errno;
280		RTE_LOG(ERR, USER1, "%s:Frag mempool creation failed for "
281			"ctx:wrk(%lu)-ctx:lcore(%u) with error code: %d\n",
282			__func__, worker, lcore, rc);
283		return rc;
284	}
285
286	tcx->frag_mpool = mp;
287
288	return 0;
289}
290
291int
292be_queue_init(struct tldk_ctx *tcx, const tldk_conf_t *cf)
293{
294	int32_t socket, rc;
295	uint16_t queue_id;
296	uint32_t port_id, i;
297	struct rte_eth_dev_info dev_info;
298	const struct tldk_ctx_conf *ctx;
299	const struct tldk_port_conf *pcf;
300
301	ctx = tcx->cf;
302	for (i = 0; i < ctx->nb_dev; i++) {
303		port_id = ctx->dev[i].port;
304		queue_id = ctx->dev[i].queue;
305		pcf = &cf->port[port_id];
306
307		rte_eth_dev_info_get(port_id, &dev_info);
308		dev_info.default_rxconf.rx_drop_en = 1;
309		dev_info.default_txconf.tx_free_thresh = TX_RING_SIZE / 2;
310
311		if (pcf->tx_offload != 0) {
312			RTE_LOG(ERR, USER1,
313				"%s(port=%u): enabling full featured TX;\n",
314				__func__, port_id);
315			dev_info.default_txconf.txq_flags = 0;
316		}
317
318		socket = rte_eth_dev_socket_id(port_id);
319
320		rc = rte_eth_rx_queue_setup(port_id, queue_id, RX_RING_SIZE,
321				socket, &dev_info.default_rxconf, tcx->mpool);
322		if (rc < 0) {
323			RTE_LOG(ERR, USER1,
324				"%s: rx queue=%u setup failed with error "
325				"code: %d\n", __func__, queue_id, rc);
326			return rc;
327		}
328
329		rc = rte_eth_tx_queue_setup(port_id, queue_id, TX_RING_SIZE,
330				socket, &dev_info.default_txconf);
331		if (rc < 0) {
332			RTE_LOG(ERR, USER1,
333				"%s: tx queue=%u setup failed with error "
334				"code: %d\n", __func__, queue_id, rc);
335			return rc;
336		}
337	}
338
339	return 0;
340}
341
342/*
343 * Setup all enabled ports.
344 */
345int
346be_port_init(tldk_conf_t *cf)
347{
348	int32_t rc;
349	uint32_t i;
350	struct tldk_port_conf *dpf;
351
352	for (i = 0; i != cf->nb_port; i++) {
353		dpf = &cf->port[i];
354		rc = port_init(dpf);
355		if (rc != 0) {
356			RTE_LOG(ERR, USER1,
357				"%s: port=%u init failed with error code: %d\n",
358				__func__, dpf->id, rc);
359			return NGX_ERROR;
360		}
361		rte_eth_macaddr_get(dpf->id, &dpf->mac);
362		rte_eth_promiscuous_enable(dpf->id);
363	}
364
365	return NGX_OK;
366}
367
368static int
369be_add_ipv4_route(struct tldk_ctx *tcx, const struct tldk_dest_conf *dcf,
370	uint8_t idx)
371{
372	int32_t rc;
373	uint32_t addr, depth;
374	char str[INET_ADDRSTRLEN];
375
376	depth = dcf->prfx;
377	addr = rte_be_to_cpu_32(dcf->ipv4.s_addr);
378
379	inet_ntop(AF_INET, &dcf->ipv4, str, sizeof(str));
380	rc = rte_lpm_add(tcx->lpm4, addr, depth, idx);
381	RTE_LOG(NOTICE, USER1, "%s(lcore=%u,dev_id=%u,dev=%p,"
382			"ipv4=%s/%u,mtu=%u,"
383			"mac=%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx) "
384			"returns %d;\n",
385			__func__, tcx->cf->lcore, dcf->dev, tcx->dst4[idx].dev,
386			str, depth, tcx->dst4[idx].mtu,
387			dcf->mac.addr_bytes[0], dcf->mac.addr_bytes[1],
388			dcf->mac.addr_bytes[2], dcf->mac.addr_bytes[3],
389			dcf->mac.addr_bytes[4], dcf->mac.addr_bytes[5],
390			rc);
391
392	return rc;
393}
394
395static int
396be_add_ipv6_route(struct tldk_ctx *tcx, const struct tldk_dest_conf *dcf,
397	uint8_t idx)
398{
399	int32_t rc;
400	uint32_t depth;
401	char str[INET6_ADDRSTRLEN];
402
403	depth = dcf->prfx;
404
405	rc = rte_lpm6_add(tcx->lpm6, (uint8_t *)(uintptr_t)dcf->ipv6.s6_addr,
406			depth, idx);
407
408	inet_ntop(AF_INET6, &dcf->ipv6, str, sizeof(str));
409	RTE_LOG(NOTICE, USER1, "%s(lcore=%u,dev_id=%u,dev=%p,"
410		"ipv6=%s/%u,mtu=%u,"
411		"mac=%02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx) "
412		"returns %d;\n",
413		__func__, tcx->cf->lcore, dcf->dev, tcx->dst6[idx].dev,
414		str, depth, tcx->dst4[idx].mtu,
415		dcf->mac.addr_bytes[0], dcf->mac.addr_bytes[1],
416		dcf->mac.addr_bytes[2], dcf->mac.addr_bytes[3],
417		dcf->mac.addr_bytes[4], dcf->mac.addr_bytes[5],
418		rc);
419
420	return rc;
421}
422
423static void
424fill_dst(struct tle_dest *dst, const struct tldk_dev *td,
425	const struct tldk_port_conf *pcf, const struct tldk_dest_conf *dest,
426	uint16_t l3_type, struct rte_mempool *mp)
427{
428	struct ether_hdr *eth;
429	struct ipv4_hdr *ip4h;
430	struct ipv6_hdr *ip6h;
431
432	dst->dev = td->dev;
433	dst->head_mp = mp;
434	dst->mtu = RTE_MIN(dest->mtu, pcf->mtu);
435	dst->l2_len = sizeof(*eth);
436
437	eth = (struct ether_hdr *)dst->hdr;
438
439	ether_addr_copy(&pcf->mac, &eth->s_addr);
440	ether_addr_copy(&dest->mac, &eth->d_addr);
441	eth->ether_type = rte_cpu_to_be_16(l3_type);
442
443	if (l3_type == ETHER_TYPE_IPv4) {
444		dst->l3_len = sizeof(*ip4h);
445		ip4h = (struct ipv4_hdr *)(eth + 1);
446		ip4h->version_ihl = 4 << 4 |
447			sizeof(*ip4h) / IPV4_IHL_MULTIPLIER;
448		ip4h->time_to_live = 64;
449		ip4h->next_proto_id = IPPROTO_TCP;
450	} else if (l3_type == ETHER_TYPE_IPv6) {
451		dst->l3_len = sizeof(*ip6h);
452		ip6h = (struct ipv6_hdr *)(eth + 1);
453		ip6h->vtc_flow = 6 << 4;
454		ip6h->proto = IPPROTO_TCP;
455		ip6h->hop_limits = 64;
456	}
457}
458
459static int
460be_add_dest(const struct tldk_dest_conf *dcf, struct tldk_ctx *tcx,
461	uint32_t dev_idx, const struct tldk_port_conf *pcf, uint32_t family,
462	uint32_t dnum)
463{
464	struct tle_dest *dp;
465	uint32_t i, n, m;
466	uint16_t l3_type;
467	int32_t rc = 0;
468
469	if (family == AF_INET) {
470		n = tcx->dst4_num;
471		dp = tcx->dst4 + n;
472		m = RTE_DIM(tcx->dst4);
473		l3_type = ETHER_TYPE_IPv4;
474	} else {
475		n = tcx->dst6_num;
476		dp = tcx->dst6 + n;
477		m = RTE_DIM(tcx->dst6);
478		l3_type = ETHER_TYPE_IPv6;
479	}
480
481	if (n + dnum >= m) {
482		RTE_LOG(ERR, USER1, "%s(lcore=%u, family=%hu, dnum=%u) exceeds "
483			"maximum allowed number of destinations(%u);\n",
484			__func__, tcx->cf->lcore, family, dnum, m);
485		return -ENOSPC;
486	}
487
488	for (i = 0; i != dnum && rc == 0; i++) {
489		fill_dst(dp + i, &tcx->dev[dev_idx], pcf, dcf,
490			l3_type, tcx->frag_mpool);
491		if (family == AF_INET)
492			rc = be_add_ipv4_route(tcx, dcf, n + i);
493		else
494			rc = be_add_ipv6_route(tcx, dcf, n + i);
495	}
496
497	if (family == AF_INET)
498		tcx->dst4_num = n + i;
499	else
500		tcx->dst6_num = n + i;
501
502	return rc;
503}
504
505int
506be_dst_init(struct tldk_ctx *tcx, const tldk_conf_t *cf)
507{
508	uint32_t i, f, d, l, port_id;
509	const struct tldk_ctx_conf *ctx_cf = tcx->cf;
510	const struct tldk_dest_conf *dcf;
511	const struct tldk_port_conf *pcf;
512	int32_t rc = 0;
513
514	for (i = 0; i < ctx_cf->nb_dest; i++) {
515		dcf = &ctx_cf->dest[i];
516		f = dcf->family;
517		d = dcf->dev;
518		for (l = 0; l != tcx->nb_dev; l++) {
519			if (tcx->dev[l].cf.id == d) {
520				/* fetch the port conf for the port
521				 * associated with device
522				 */
523				port_id = tcx->dev[l].cf.port;
524				pcf = &cf->port[port_id];
525				rc = be_add_dest(dcf, tcx, l, pcf, f, 1);
526				if (rc != 0) {
527					RTE_LOG(ERR, USER1,
528						"%s(tcx=%u, family=%u) "
529						"could not add "
530						"destinations(%u)\n",
531						__func__, ctx_cf->lcore, f, i);
532					return -ENOSPC;
533				}
534				break;
535			}
536		}
537	}
538
539	return rc;
540}
541
542int
543be_add_dev(struct tldk_ctx *tcx, const tldk_conf_t *cf)
544{
545	int32_t rc = 0;
546	uint32_t i, port_id;
547	struct tle_dev_param dprm;
548	const struct tldk_port_conf *pcf;
549
550	memset(&dprm, 0, sizeof(dprm));
551
552	/* add the tle_dev on all applicable ports of the context */
553	for (i = 0; i != tcx->cf->nb_dev; i++) {
554
555		/* get the port id associated with the device */
556		port_id = tcx->cf->dev[i].port;
557
558		/* get the port config by port id */
559		pcf = &cf->port[port_id];
560
561		/* populate the tle_dev_param struct */
562		dprm.rx_offload = pcf->rx_offload;
563		dprm.tx_offload = pcf->tx_offload;
564		dprm.local_addr4.s_addr = pcf->ipv4;
565
566		memcpy(&dprm.local_addr6, &pcf->ipv6,
567			sizeof(pcf->ipv6));
568
569		/* add the tle_dev */
570		tcx->dev[i].dev = tle_add_dev(tcx->ctx, &dprm);
571
572		RTE_LOG(NOTICE, USER1, "%s(port=%u), dev: %p\n",
573			__func__, port_id,
574			tcx->dev[i].dev);
575
576		if (tcx->dev[i].dev == NULL)
577			rc = -rte_errno;
578
579		if (rc != 0)
580			return rc;
581
582		tcx->nb_dev++;
583		tcx->dev[i].cf = tcx->cf->dev[i];
584	}
585
586	return rc;
587}
588
589static uint32_t
590get_ptypes(const struct tldk_dev *td)
591{
592	uint32_t smask;
593	int32_t i, rc;
594	const uint32_t pmask = RTE_PTYPE_L2_MASK | RTE_PTYPE_L3_MASK |
595		RTE_PTYPE_L4_MASK;
596
597	smask = 0;
598	rc = rte_eth_dev_get_supported_ptypes(td->cf.port, pmask, NULL, 0);
599	if (rc < 0) {
600		RTE_LOG(ERR, USER1,
601			"%s(port=%u) failed to get supported ptypes;\n",
602			__func__, td->cf.port);
603		return smask;
604	}
605
606	uint32_t ptype[rc];
607	rc = rte_eth_dev_get_supported_ptypes(td->cf.port, pmask, ptype, rc);
608
609	for (i = 0; i != rc; i++) {
610		switch (ptype[i]) {
611		case RTE_PTYPE_L2_ETHER:
612			smask |= ETHER_PTYPE;
613			break;
614		case RTE_PTYPE_L3_IPV4:
615		case RTE_PTYPE_L3_IPV4_EXT_UNKNOWN:
616			smask |= IPV4_PTYPE;
617			break;
618		case RTE_PTYPE_L3_IPV4_EXT:
619			smask |= IPV4_EXT_PTYPE;
620			break;
621		case RTE_PTYPE_L3_IPV6:
622		case RTE_PTYPE_L3_IPV6_EXT_UNKNOWN:
623			smask |= IPV6_PTYPE;
624			break;
625		case RTE_PTYPE_L3_IPV6_EXT:
626			smask |= IPV6_EXT_PTYPE;
627			break;
628		case RTE_PTYPE_L4_TCP:
629			smask |= TCP_PTYPE;
630			break;
631		case RTE_PTYPE_L4_UDP:
632			smask |= UDP_PTYPE;
633			break;
634		}
635	}
636
637	return smask;
638}
639
640static inline uint64_t
641_mbuf_tx_offload(uint64_t il2, uint64_t il3, uint64_t il4, uint64_t tso,
642	uint64_t ol3, uint64_t ol2)
643{
644	return il2 | il3 << 7 | il4 << 16 | tso << 24 | ol3 << 40 | ol2 << 49;
645}
646
647static inline void
648fill_pkt_hdr_len(struct rte_mbuf *m, uint32_t l2, uint32_t l3, uint32_t l4)
649{
650	m->tx_offload = _mbuf_tx_offload(l2, l3, l4, 0, 0, 0);
651}
652
653static inline int
654is_ipv4_frag(const struct ipv4_hdr *iph)
655{
656	const uint16_t mask = rte_cpu_to_be_16(~IPV4_HDR_DF_FLAG);
657
658	return ((mask & iph->fragment_offset) != 0);
659}
660
661static inline uint32_t
662get_tcp_header_size(struct rte_mbuf *m, uint32_t l2_len, uint32_t l3_len)
663{
664	const struct tcp_hdr *tcp;
665
666	tcp = rte_pktmbuf_mtod_offset(m, struct tcp_hdr *, l2_len + l3_len);
667	return (tcp->data_off >> 4) * 4;
668}
669
670static inline void
671adjust_ipv4_pktlen(struct rte_mbuf *m, uint32_t l2_len)
672{
673	uint32_t plen, trim;
674	const struct ipv4_hdr *iph;
675
676	iph = rte_pktmbuf_mtod_offset(m, const struct ipv4_hdr *, l2_len);
677	plen = rte_be_to_cpu_16(iph->total_length) + l2_len;
678	if (plen < m->pkt_len) {
679		trim = m->pkt_len - plen;
680		rte_pktmbuf_trim(m, trim);
681	}
682}
683
684static inline void
685adjust_ipv6_pktlen(struct rte_mbuf *m, uint32_t l2_len)
686{
687	uint32_t plen, trim;
688	const struct ipv6_hdr *iph;
689
690	iph = rte_pktmbuf_mtod_offset(m, const struct ipv6_hdr *, l2_len);
691	plen = rte_be_to_cpu_16(iph->payload_len) + sizeof(*iph) + l2_len;
692	if (plen < m->pkt_len) {
693		trim = m->pkt_len - plen;
694		rte_pktmbuf_trim(m, trim);
695	}
696}
697
698static inline void
699tcp_stat_update(struct tldk_ctx *lc, const struct rte_mbuf *m,
700	uint32_t l2_len, uint32_t l3_len)
701{
702	const struct tcp_hdr *th;
703
704	th = rte_pktmbuf_mtod_offset(m, struct tcp_hdr *, l2_len + l3_len);
705	lc->tcp_stat.flags[th->tcp_flags]++;
706}
707
708static inline uint32_t
709get_ipv4_hdr_len(struct rte_mbuf *m, uint32_t l2, uint32_t proto, uint32_t frag)
710{
711	const struct ipv4_hdr *iph;
712	int32_t dlen, len;
713
714	dlen = rte_pktmbuf_data_len(m);
715	dlen -= l2;
716
717	iph = rte_pktmbuf_mtod_offset(m, const struct ipv4_hdr *, l2);
718	len = (iph->version_ihl & IPV4_HDR_IHL_MASK) * IPV4_IHL_MULTIPLIER;
719
720	if (frag != 0 && is_ipv4_frag(iph)) {
721		m->packet_type &= ~RTE_PTYPE_L4_MASK;
722		m->packet_type |= RTE_PTYPE_L4_FRAG;
723	}
724
725	if (len > dlen || (proto <= IPPROTO_MAX && iph->next_proto_id != proto))
726		m->packet_type = RTE_PTYPE_UNKNOWN;
727
728	return len;
729}
730
731static inline int
732ipv6x_hdr(uint32_t proto)
733{
734	return (proto == IPPROTO_HOPOPTS ||
735		proto == IPPROTO_ROUTING ||
736		proto == IPPROTO_FRAGMENT ||
737		proto == IPPROTO_AH ||
738		proto == IPPROTO_NONE ||
739		proto == IPPROTO_DSTOPTS);
740}
741
742static inline uint32_t
743get_ipv6x_hdr_len(struct rte_mbuf *m, uint32_t l2, uint32_t nproto,
744	uint32_t fproto)
745{
746	const struct ip6_ext *ipx;
747	int32_t dlen, len, ofs;
748
749	len = sizeof(struct ipv6_hdr);
750
751	dlen = rte_pktmbuf_data_len(m);
752	dlen -= l2;
753
754	ofs = l2 + len;
755	ipx = rte_pktmbuf_mtod_offset(m, const struct ip6_ext *, ofs);
756
757	while (ofs > 0 && len < dlen) {
758
759		switch (nproto) {
760		case IPPROTO_HOPOPTS:
761		case IPPROTO_ROUTING:
762		case IPPROTO_DSTOPTS:
763			ofs = (ipx->ip6e_len + 1) << 3;
764			break;
765		case IPPROTO_AH:
766			ofs = (ipx->ip6e_len + 2) << 2;
767			break;
768		case IPPROTO_FRAGMENT:
769			/*
770			 * tso_segsz is not used by RX, so use it as temporary
771			 * buffer to store the fragment offset.
772			 */
773			m->tso_segsz = ofs;
774			ofs = sizeof(struct ip6_frag);
775			m->packet_type &= ~RTE_PTYPE_L4_MASK;
776			m->packet_type |= RTE_PTYPE_L4_FRAG;
777			break;
778		default:
779			ofs = 0;
780		}
781
782		if (ofs > 0) {
783			nproto = ipx->ip6e_nxt;
784			len += ofs;
785			ipx += ofs / sizeof(*ipx);
786		}
787	}
788
789	/* unrecognized or invalid packet. */
790	if ((ofs == 0 && nproto != fproto) || len > dlen)
791		m->packet_type = RTE_PTYPE_UNKNOWN;
792
793	return len;
794}
795
796static inline uint32_t
797get_ipv6_hdr_len(struct rte_mbuf *m, uint32_t l2, uint32_t fproto)
798{
799	const struct ipv6_hdr *iph;
800
801	iph = rte_pktmbuf_mtod_offset(m, const struct ipv6_hdr *,
802		sizeof(struct ether_hdr));
803
804	if (iph->proto == fproto)
805		return sizeof(struct ipv6_hdr);
806	else if (ipv6x_hdr(iph->proto) != 0)
807		return get_ipv6x_hdr_len(m, l2, iph->proto, fproto);
808
809	m->packet_type = RTE_PTYPE_UNKNOWN;
810	return 0;
811}
812
813static inline void
814fill_eth_tcp_hdr_len(struct rte_mbuf *m)
815{
816	uint32_t dlen, l2_len, l3_len, l4_len;
817	uint16_t etp;
818	const struct ether_hdr *eth;
819
820	dlen = rte_pktmbuf_data_len(m);
821
822	/* check that first segment is at least 54B long. */
823	if (dlen < sizeof(struct ether_hdr) + sizeof(struct ipv4_hdr) +
824			sizeof(struct tcp_hdr)) {
825		m->packet_type = RTE_PTYPE_UNKNOWN;
826		return;
827	}
828
829	l2_len = sizeof(*eth);
830
831	eth = rte_pktmbuf_mtod(m, const struct ether_hdr *);
832	etp = eth->ether_type;
833	if (etp == rte_be_to_cpu_16(ETHER_TYPE_VLAN))
834		l2_len += sizeof(struct vlan_hdr);
835
836	if (etp == rte_be_to_cpu_16(ETHER_TYPE_IPv4)) {
837		m->packet_type = RTE_PTYPE_L4_TCP |
838			RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
839			RTE_PTYPE_L2_ETHER;
840		l3_len = get_ipv4_hdr_len(m, l2_len, IPPROTO_TCP, 1);
841		l4_len = get_tcp_header_size(m, l2_len, l3_len);
842		fill_pkt_hdr_len(m, l2_len, l3_len, l4_len);
843		adjust_ipv4_pktlen(m, l2_len);
844	} else if (etp == rte_be_to_cpu_16(ETHER_TYPE_IPv6) &&
845			dlen >= l2_len + sizeof(struct ipv6_hdr) +
846			sizeof(struct tcp_hdr)) {
847		m->packet_type = RTE_PTYPE_L4_TCP |
848			RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
849			RTE_PTYPE_L2_ETHER;
850		l3_len = get_ipv6_hdr_len(m, l2_len, IPPROTO_TCP);
851		l4_len = get_tcp_header_size(m, l2_len, l3_len);
852		fill_pkt_hdr_len(m, l2_len, l3_len, l4_len);
853		adjust_ipv6_pktlen(m, l2_len);
854	} else
855		m->packet_type = RTE_PTYPE_UNKNOWN;
856}
857
858/*
859 * HW can recognize L2/L3 with/without extensions/L4 (ixgbe/igb/fm10k)
860 */
861static uint16_t
862type0_tcp_rx_callback(__rte_unused dpdk_port_t port,
863	__rte_unused uint16_t queue,
864	struct rte_mbuf *pkt[], uint16_t nb_pkts,
865	__rte_unused uint16_t max_pkts, __rte_unused void *user_param)
866{
867	uint32_t j, tp;
868	uint32_t l4_len, l3_len, l2_len;
869	const struct ether_hdr *eth;
870
871	l2_len = sizeof(*eth);
872
873	for (j = 0; j != nb_pkts; j++) {
874
875		BE_PKT_DUMP(pkt[j]);
876
877		tp = pkt[j]->packet_type & (RTE_PTYPE_L4_MASK |
878			RTE_PTYPE_L3_MASK | RTE_PTYPE_L2_MASK);
879
880		switch (tp) {
881		/* non fragmented tcp packets. */
882		case (RTE_PTYPE_L4_TCP | RTE_PTYPE_L3_IPV4 |
883				RTE_PTYPE_L2_ETHER):
884			l4_len = get_tcp_header_size(pkt[j], l2_len,
885				sizeof(struct ipv4_hdr));
886			fill_pkt_hdr_len(pkt[j], l2_len,
887				sizeof(struct ipv4_hdr), l4_len);
888			adjust_ipv4_pktlen(pkt[j], l2_len);
889			break;
890		case (RTE_PTYPE_L4_TCP | RTE_PTYPE_L3_IPV6 |
891				RTE_PTYPE_L2_ETHER):
892			l4_len = get_tcp_header_size(pkt[j], l2_len,
893				sizeof(struct ipv6_hdr));
894			fill_pkt_hdr_len(pkt[j], l2_len,
895				sizeof(struct ipv6_hdr), l4_len);
896			adjust_ipv6_pktlen(pkt[j], l2_len);
897			break;
898		case (RTE_PTYPE_L4_TCP | RTE_PTYPE_L3_IPV4_EXT |
899				RTE_PTYPE_L2_ETHER):
900			l3_len = get_ipv4_hdr_len(pkt[j], l2_len,
901				IPPROTO_TCP, 0);
902			l4_len = get_tcp_header_size(pkt[j], l2_len, l3_len);
903			fill_pkt_hdr_len(pkt[j], l2_len, l3_len, l4_len);
904			adjust_ipv4_pktlen(pkt[j], l2_len);
905			break;
906		case (RTE_PTYPE_L4_TCP | RTE_PTYPE_L3_IPV6_EXT |
907				RTE_PTYPE_L2_ETHER):
908			l3_len = get_ipv6_hdr_len(pkt[j], l2_len, IPPROTO_TCP);
909			l4_len = get_tcp_header_size(pkt[j], l2_len, l3_len);
910			fill_pkt_hdr_len(pkt[j], l2_len, l3_len, l4_len);
911			adjust_ipv6_pktlen(pkt[j], l2_len);
912			break;
913		default:
914			/* treat packet types as invalid. */
915			pkt[j]->packet_type = RTE_PTYPE_UNKNOWN;
916			break;
917		}
918	}
919
920	return nb_pkts;
921}
922
923/*
924 * HW can recognize L2/L3/L4 and fragments (i40e).
925 */
926static uint16_t
927type1_tcp_rx_callback(__rte_unused dpdk_port_t port,
928	__rte_unused uint16_t queue,
929	struct rte_mbuf *pkt[], uint16_t nb_pkts,
930	__rte_unused uint16_t max_pkts, void *user_param)
931{
932	uint32_t j, tp;
933	struct tldk_ctx *tcx;
934	uint32_t l4_len, l3_len, l2_len;
935	const struct ether_hdr *eth;
936
937	tcx = user_param;
938	l2_len = sizeof(*eth);
939
940	for (j = 0; j != nb_pkts; j++) {
941
942		BE_PKT_DUMP(pkt[j]);
943
944		tp = pkt[j]->packet_type & (RTE_PTYPE_L4_MASK |
945			RTE_PTYPE_L3_MASK | RTE_PTYPE_L2_MASK);
946
947		switch (tp) {
948		case (RTE_PTYPE_L4_TCP | RTE_PTYPE_L3_IPV4_EXT_UNKNOWN |
949				RTE_PTYPE_L2_ETHER):
950			l3_len = get_ipv4_hdr_len(pkt[j], l2_len,
951				IPPROTO_TCP, 0);
952			l4_len = get_tcp_header_size(pkt[j], l2_len, l3_len);
953			fill_pkt_hdr_len(pkt[j], l2_len, l3_len, l4_len);
954			adjust_ipv4_pktlen(pkt[j], l2_len);
955			tcp_stat_update(tcx, pkt[j], l2_len, l3_len);
956			break;
957		case (RTE_PTYPE_L4_TCP | RTE_PTYPE_L3_IPV6_EXT_UNKNOWN |
958				RTE_PTYPE_L2_ETHER):
959			l3_len = get_ipv6_hdr_len(pkt[j], l2_len, IPPROTO_TCP);
960			l4_len = get_tcp_header_size(pkt[j], l2_len, l3_len);
961			fill_pkt_hdr_len(pkt[j], l2_len, l3_len, l4_len);
962			adjust_ipv6_pktlen(pkt[j], l2_len);
963			tcp_stat_update(tcx, pkt[j], l2_len, l3_len);
964			break;
965		default:
966			/* treat packet types as invalid. */
967			pkt[j]->packet_type = RTE_PTYPE_UNKNOWN;
968			break;
969		}
970
971	}
972
973	return nb_pkts;
974}
975
976static uint16_t
977typen_tcp_rx_callback(__rte_unused dpdk_port_t port,
978	__rte_unused uint16_t queue,
979	struct rte_mbuf *pkt[], uint16_t nb_pkts,
980	__rte_unused uint16_t max_pkts, __rte_unused void *user_param)
981{
982	uint32_t j;
983
984	for (j = 0; j != nb_pkts; j++) {
985
986		BE_PKT_DUMP(pkt[j]);
987		fill_eth_tcp_hdr_len(pkt[j]);
988	}
989
990	return nb_pkts;
991}
992
993int
994setup_rx_cb(const struct tldk_dev *td, struct tldk_ctx *tcx)
995{
996	int32_t rc;
997	uint32_t i, n, smask;
998	void *cb;
999	const struct ptype2cb *ptype2cb;
1000
1001	static const struct ptype2cb tcp_ptype2cb[] = {
1002		{
1003			.mask = ETHER_PTYPE | IPV4_PTYPE | IPV4_EXT_PTYPE |
1004				IPV6_PTYPE | IPV6_EXT_PTYPE | TCP_PTYPE,
1005			.name = "HW l2/l3x/l4-tcp ptype",
1006			.fn = type0_tcp_rx_callback,
1007		},
1008		{
1009			.mask = ETHER_PTYPE | IPV4_PTYPE | IPV6_PTYPE |
1010				TCP_PTYPE,
1011			.name = "HW l2/l3/l4-tcp ptype",
1012			.fn = type1_tcp_rx_callback,
1013		},
1014		{
1015			.mask = 0,
1016			.name = "tcp no HW ptype",
1017			.fn = typen_tcp_rx_callback,
1018		},
1019	};
1020
1021	smask = get_ptypes(td);
1022
1023	ptype2cb = tcp_ptype2cb;
1024	n = RTE_DIM(tcp_ptype2cb);
1025
1026	for (i = 0; i != n; i++) {
1027		if ((smask & ptype2cb[i].mask) == ptype2cb[i].mask) {
1028			cb = rte_eth_add_rx_callback(td->cf.port, td->cf.queue,
1029				ptype2cb[i].fn, tcx);
1030			rc = -rte_errno;
1031			RTE_LOG(ERR, USER1,
1032				"%s(port=%u), setup RX callback \"%s\" "
1033				"returns %p;\n",
1034				__func__, td->cf.port,  ptype2cb[i].name, cb);
1035				return ((cb == NULL) ? rc : 0);
1036		}
1037	}
1038
1039	/* no proper callback found. */
1040	RTE_LOG(ERR, USER1,
1041		"%s(port=%u) failed to find an appropriate callback;\n",
1042		__func__, td->cf.port);
1043	return -ENOENT;
1044}
1045
1046int
1047be_lcore_setup(struct tldk_ctx *tcx)
1048{
1049	uint32_t i;
1050	int32_t rc;
1051
1052	RTE_LOG(NOTICE, USER1, "%s:(lcore=%u, ctx=%p) start\n",
1053		__func__, tcx->cf->lcore, tcx->ctx);
1054
1055	rc = 0;
1056	for (i = 0; i != tcx->nb_dev && rc == 0; i++) {
1057		RTE_LOG(NOTICE, USER1, "%s:%u(port=%u, q=%u)\n",
1058			__func__, i, tcx->dev[i].cf.port, tcx->dev[i].cf.queue);
1059
1060		rc = setup_rx_cb(&tcx->dev[i], tcx);
1061		if (rc < 0)
1062			return rc;
1063	}
1064
1065	return rc;
1066}
1067
1068static inline void
1069be_rx(struct tldk_dev *dev)
1070{
1071	uint32_t j, k, n;
1072	struct rte_mbuf *pkt[MAX_PKT_BURST];
1073	struct rte_mbuf *rp[MAX_PKT_BURST];
1074	int32_t rc[MAX_PKT_BURST];
1075
1076	n = rte_eth_rx_burst(dev->cf.port,
1077		dev->cf.queue, pkt, RTE_DIM(pkt));
1078
1079	if (n != 0) {
1080		dev->rx_stat.in += n;
1081		BE_TRACE("%s(%u): rte_eth_rx_burst(%u, %u) returns %u\n",
1082			__func__, dev->cf.id, dev->cf.port,
1083			dev->cf.queue, n);
1084
1085		k = tle_tcp_rx_bulk(dev->dev, pkt, rp, rc, n);
1086
1087		dev->rx_stat.up += k;
1088		dev->rx_stat.drop += n - k;
1089		BE_TRACE("%s: tle_tcp_rx_bulk(%p, %u) returns %u\n",
1090			__func__, dev->dev, n, k);
1091
1092		for (j = 0; j != n - k; j++) {
1093			BE_TRACE("%s:%d(port=%u) rp[%u]={%p, %d};\n",
1094				__func__, __LINE__, dev->cf.port,
1095				j, rp[j], rc[j]);
1096			rte_pktmbuf_free(rp[j]);
1097		}
1098	}
1099}
1100
1101static inline void
1102be_tx(struct tldk_dev *dev)
1103{
1104	uint32_t j = 0, k, n;
1105	struct rte_mbuf **mb;
1106
1107	n = dev->tx_buf.num;
1108	k = RTE_DIM(dev->tx_buf.pkt) - n;
1109	mb = dev->tx_buf.pkt;
1110
1111	if (k >= RTE_DIM(dev->tx_buf.pkt) / 2) {
1112		j = tle_tcp_tx_bulk(dev->dev, mb + n, k);
1113		n += j;
1114		dev->tx_stat.down += j;
1115	}
1116
1117	if (n == 0)
1118		return;
1119
1120	BE_TRACE("%s: tle_tcp_tx_bulk(%p) returns %u,\n"
1121		"total pkts to send: %u\n",
1122		__func__, dev->dev, j, n);
1123
1124	for (j = 0; j != n; j++)
1125		BE_PKT_DUMP(mb[j]);
1126
1127	k = rte_eth_tx_burst(dev->cf.port,
1128			dev->cf.queue, mb, n);
1129
1130	dev->tx_stat.out += k;
1131	dev->tx_stat.drop += n - k;
1132	BE_TRACE("%s: rte_eth_tx_burst(%u, %u, %u) returns %u\n",
1133		__func__, dev->cf.port,
1134		dev->cf.queue, n, k);
1135
1136	dev->tx_buf.num = n - k;
1137	if (k != 0)
1138		for (j = k; j != n; j++)
1139			mb[j - k] = mb[j];
1140}
1141
1142void
1143be_lcore_tcp(struct tldk_ctx *tcx)
1144{
1145	uint32_t i;
1146
1147	if (tcx == NULL)
1148		return;
1149
1150	for (i = 0; i != tcx->nb_dev; i++) {
1151		be_rx(&tcx->dev[i]);
1152		be_tx(&tcx->dev[i]);
1153	}
1154	tle_tcp_process(tcx->ctx, TCP_MAX_PROCESS);
1155}
1156
1157void
1158be_lcore_clear(struct tldk_ctx *tcx)
1159{
1160	uint32_t i, j;
1161
1162	if (tcx == NULL)
1163		return;
1164
1165	RTE_LOG(NOTICE, USER1, "%s(lcore=%u, ctx: %p) finish\n",
1166		__func__, tcx->cf->lcore, tcx->ctx);
1167	for (i = 0; i != tcx->nb_dev; i++) {
1168		RTE_LOG(NOTICE, USER1, "%s:%u(port=%u, q=%u, lcore=%u, dev=%p) "
1169			"rx_stats={"
1170			"in=%" PRIu64 ",up=%" PRIu64 ",drop=%" PRIu64 "}, "
1171			"tx_stats={"
1172			"in=%" PRIu64 ",up=%" PRIu64 ",drop=%" PRIu64 "};\n",
1173			__func__, i, tcx->dev[i].cf.port, tcx->dev[i].cf.queue,
1174			tcx->cf->lcore,
1175			tcx->dev[i].dev,
1176			tcx->dev[i].rx_stat.in,
1177			tcx->dev[i].rx_stat.up,
1178			tcx->dev[i].rx_stat.drop,
1179			tcx->dev[i].tx_stat.down,
1180			tcx->dev[i].tx_stat.out,
1181			tcx->dev[i].tx_stat.drop);
1182	}
1183
1184	RTE_LOG(NOTICE, USER1, "tcp_stat={\n");
1185	for (i = 0; i != RTE_DIM(tcx->tcp_stat.flags); i++) {
1186		if (tcx->tcp_stat.flags[i] != 0)
1187			RTE_LOG(NOTICE, USER1, "[flag=%#x]==%" PRIu64 ";\n",
1188				i, tcx->tcp_stat.flags[i]);
1189	}
1190	RTE_LOG(NOTICE, USER1, "};\n");
1191
1192	for (i = 0; i != tcx->nb_dev; i++)
1193		for (j = 0; j != tcx->dev[i].tx_buf.num; j++)
1194			rte_pktmbuf_free(tcx->dev[i].tx_buf.pkt[j]);
1195
1196}
1197
1198void
1199be_stop_port(uint32_t port)
1200{
1201	struct rte_eth_stats stats;
1202
1203	RTE_LOG(NOTICE, USER1, "%s: stoping port %u\n", __func__, port);
1204
1205	rte_eth_stats_get(port, &stats);
1206	RTE_LOG(NOTICE, USER1, "port %u stats={\n"
1207		"ipackets=%" PRIu64 ";"
1208		"ibytes=%" PRIu64 ";"
1209		"ierrors=%" PRIu64 ";"
1210		"imissed=%" PRIu64 ";\n"
1211		"opackets=%" PRIu64 ";"
1212		"obytes=%" PRIu64 ";"
1213		"oerrors=%" PRIu64 ";\n"
1214		"}\n",
1215		port,
1216		stats.ipackets,
1217		stats.ibytes,
1218		stats.ierrors,
1219		stats.imissed,
1220		stats.opackets,
1221		stats.obytes,
1222		stats.oerrors);
1223	rte_eth_dev_stop(port);
1224}
1225
1226int
1227be_lcore_main(void *arg)
1228{
1229	int32_t rc;
1230	uint32_t lid, i;
1231	struct tldk_ctx *tcx;
1232	struct lcore_ctxs_list *lc_ctx;
1233
1234	lc_ctx = arg;
1235	lid = rte_lcore_id();
1236
1237	RTE_LOG(NOTICE, USER1, "%s(lcore=%u) start\n", __func__, lid);
1238
1239	rc = 0;
1240	while (force_quit == 0) {
1241		for (i = 0; i < lc_ctx->nb_ctxs; i++) {
1242			tcx = lc_ctx->ctxs[i];
1243			be_lcore_tcp(tcx);
1244		}
1245	}
1246
1247	RTE_LOG(NOTICE, USER1, "%s(lcore=%u) finish\n", __func__, lid);
1248
1249	return rc;
1250}
1251