1/*
2 * Copyright (c) 2016  Intel Corporation.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 *     http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#ifndef PORT_H_
17#define PORT_H_
18
19static void
20prepare_hash_key(struct netbe_port *uprt, uint8_t key_size, uint16_t family)
21{
22	uint32_t align_nb_q;
23
24	align_nb_q = rte_align32pow2(uprt->nb_lcore);
25	memset(uprt->hash_key, 0, RSS_HASH_KEY_LENGTH);
26	uprt->hash_key_size = key_size;
27	if (family == AF_INET)
28		uprt->hash_key[RSS_HASH_KEY_DEST_PORT_LOC_IPV4] = align_nb_q;
29	else
30		uprt->hash_key[RSS_HASH_KEY_DEST_PORT_LOC_IPV6] = align_nb_q;
31}
32
33static int
34update_rss_conf(struct netbe_port *uprt,
35	const struct rte_eth_dev_info *dev_info,
36	struct rte_eth_conf *port_conf, uint32_t proto)
37{
38	uint8_t hash_key_size;
39
40	if (uprt->nb_lcore > 1) {
41		if (dev_info->hash_key_size > 0)
42			hash_key_size = dev_info->hash_key_size;
43		else {
44			RTE_LOG(ERR, USER1,
45				"%s: dev_info did not provide a valid hash "
46				"key size\n", __func__);
47			return -EINVAL;
48		}
49
50		if (uprt->ipv4 != INADDR_ANY &&
51				memcmp(&uprt->ipv6, &in6addr_any,
52				sizeof(uprt->ipv6)) != 0) {
53			RTE_LOG(ERR, USER1,
54				"%s: RSS for both IPv4 and IPv6 not "
55				"supported!\n", __func__);
56			return -EINVAL;
57		} else if (uprt->ipv4 != INADDR_ANY) {
58			prepare_hash_key(uprt, hash_key_size, AF_INET);
59		} else if (memcmp(&uprt->ipv6, &in6addr_any, sizeof(uprt->ipv6))
60				!= 0) {
61			prepare_hash_key(uprt, hash_key_size, AF_INET6);
62		} else {
63			RTE_LOG(ERR, USER1,
64				"%s: No IPv4 or IPv6 address is found!\n",
65				__func__);
66			return -EINVAL;
67		}
68		port_conf->rxmode.mq_mode = ETH_MQ_RX_RSS;
69		if (proto == TLE_PROTO_TCP)
70			port_conf->rx_adv_conf.rss_conf.rss_hf = ETH_RSS_TCP;
71		else
72			port_conf->rx_adv_conf.rss_conf.rss_hf = ETH_RSS_UDP;
73		port_conf->rx_adv_conf.rss_conf.rss_key_len = hash_key_size;
74		port_conf->rx_adv_conf.rss_conf.rss_key = uprt->hash_key;
75	}
76
77	return 0;
78}
79
80static uint32_t
81qidx_from_hash_index(uint32_t hash, uint32_t align_nb_q)
82{
83	uint32_t i, nb_bit, q;
84
85	nb_bit = (sizeof(uint32_t) * CHAR_BIT) - __builtin_clz(align_nb_q - 1);
86	q = (hash & 1);
87	for (i = 1; i < nb_bit; i++) {
88		hash >>= 1;
89		q <<= 1;
90		q |= (hash & 1);
91	}
92
93	return q;
94}
95
96static int
97update_rss_reta(struct netbe_port *uprt,
98	const struct rte_eth_dev_info *dev_info)
99{
100	struct rte_eth_rss_reta_entry64 reta_conf[RSS_RETA_CONF_ARRAY_SIZE];
101	int32_t i, rc, align_nb_q;
102	int32_t q_index, idx, shift;
103
104	if (uprt->nb_lcore > 1) {
105		if (dev_info->reta_size == 0) {
106			RTE_LOG(ERR, USER1,
107				"%s: Redirection table size 0 is invalid for "
108				"RSS\n", __func__);
109			return -EINVAL;
110		}
111		RTE_LOG(NOTICE, USER1,
112			"%s: The reta size of port %d is %u\n",
113			__func__, uprt->id, dev_info->reta_size);
114
115		if (dev_info->reta_size > ETH_RSS_RETA_SIZE_512) {
116			RTE_LOG(ERR, USER1,
117				"%s: More than %u entries of Reta not supported\n",
118				__func__, ETH_RSS_RETA_SIZE_512);
119			return -EINVAL;
120		}
121
122		memset(reta_conf, 0, sizeof(reta_conf));
123		align_nb_q = rte_align32pow2(uprt->nb_lcore);
124		for (i = 0; i < align_nb_q; i++) {
125			q_index = qidx_from_hash_index(i, align_nb_q) %
126						uprt->nb_lcore;
127
128			idx = i / RTE_RETA_GROUP_SIZE;
129			shift = i % RTE_RETA_GROUP_SIZE;
130			reta_conf[idx].mask |= (1ULL << shift);
131			reta_conf[idx].reta[shift] = q_index;
132			RTE_LOG(NOTICE, USER1,
133				"%s: port=%u RSS reta conf: hash=%u, q=%u\n",
134				__func__, uprt->id, i, q_index);
135		}
136
137		rc = rte_eth_dev_rss_reta_update(uprt->id,
138				reta_conf, dev_info->reta_size);
139		if (rc != 0) {
140			RTE_LOG(ERR, USER1,
141				"%s: Bad redirection table parameter, "
142				"rc = %d\n", __func__, rc);
143			return rc;
144		}
145	}
146
147	return 0;
148}
149
150/*
151 * Initilise DPDK port.
152 * In current version, multi-queue per port is used.
153 */
154static int
155port_init(struct netbe_port *uprt, uint32_t proto)
156{
157	int32_t rc;
158	struct rte_eth_conf port_conf;
159	struct rte_eth_dev_info dev_info;
160
161	rte_eth_dev_info_get(uprt->id, &dev_info);
162	if ((dev_info.rx_offload_capa & uprt->rx_offload) != uprt->rx_offload) {
163		RTE_LOG(ERR, USER1,
164			"port#%u supported/requested RX offloads don't match, "
165			"supported: %#" PRIx64 ", requested: %#" PRIx64 ";\n",
166			uprt->id, (uint64_t)dev_info.rx_offload_capa,
167			(uint64_t)uprt->rx_offload);
168		return -EINVAL;
169	}
170	if ((dev_info.tx_offload_capa & uprt->tx_offload) != uprt->tx_offload) {
171		RTE_LOG(ERR, USER1,
172			"port#%u supported/requested TX offloads don't match, "
173			"supported: %#" PRIx64 ", requested: %#" PRIx64 ";\n",
174			uprt->id, (uint64_t)dev_info.tx_offload_capa,
175			(uint64_t)uprt->tx_offload);
176		return -EINVAL;
177	}
178
179	port_conf = port_conf_default;
180	if ((uprt->rx_offload & RX_CSUM_OFFLOAD) != 0) {
181		RTE_LOG(ERR, USER1, "%s(%u): enabling RX csum offload;\n",
182			__func__, uprt->id);
183		port_conf.rxmode.offloads |= uprt->rx_offload & RX_CSUM_OFFLOAD;
184	}
185	port_conf.rxmode.max_rx_pkt_len = uprt->mtu + ETHER_CRC_LEN;
186	if (port_conf.rxmode.max_rx_pkt_len > ETHER_MAX_LEN)
187		port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
188
189	rc = update_rss_conf(uprt, &dev_info, &port_conf, proto);
190	if (rc != 0)
191		return rc;
192
193	port_conf.txmode.offloads = uprt->tx_offload;
194
195	rc = rte_eth_dev_configure(uprt->id, uprt->nb_lcore, uprt->nb_lcore,
196			&port_conf);
197	RTE_LOG(NOTICE, USER1,
198		"%s: rte_eth_dev_configure(prt_id=%u, nb_rxq=%u, nb_txq=%u) "
199		"returns %d;\n", __func__, uprt->id, uprt->nb_lcore,
200		uprt->nb_lcore, rc);
201	if (rc != 0)
202		return rc;
203
204	return 0;
205}
206
207static int
208queue_init(struct netbe_port *uprt, struct rte_mempool *mp)
209{
210	int32_t socket, rc;
211	uint16_t q;
212	uint32_t nb_rxd, nb_txd;
213	struct rte_eth_dev_info dev_info;
214
215	rte_eth_dev_info_get(uprt->id, &dev_info);
216
217	socket = rte_eth_dev_socket_id(uprt->id);
218
219	dev_info.default_rxconf.rx_drop_en = 1;
220
221	nb_rxd = RTE_MIN(RX_RING_SIZE, dev_info.rx_desc_lim.nb_max);
222	nb_txd = RTE_MIN(TX_RING_SIZE, dev_info.tx_desc_lim.nb_max);
223
224	dev_info.default_txconf.tx_free_thresh = nb_txd / 2;
225
226	for (q = 0; q < uprt->nb_lcore; q++) {
227		rc = rte_eth_rx_queue_setup(uprt->id, q, nb_rxd,
228			socket, &dev_info.default_rxconf, mp);
229		if (rc < 0) {
230			RTE_LOG(ERR, USER1,
231				"%s: rx queue=%u setup failed with error "
232				"code: %d\n", __func__, q, rc);
233			return rc;
234		}
235	}
236
237	for (q = 0; q < uprt->nb_lcore; q++) {
238		rc = rte_eth_tx_queue_setup(uprt->id, q, nb_txd,
239			socket, &dev_info.default_txconf);
240		if (rc < 0) {
241			RTE_LOG(ERR, USER1,
242				"%s: tx queue=%u setup failed with error "
243				"code: %d\n", __func__, q, rc);
244			return rc;
245		}
246	}
247	return 0;
248}
249
250/*
251 * Check that lcore is enabled, not master, and not in use already.
252 */
253static int
254check_lcore(uint32_t lc)
255{
256	if (rte_lcore_is_enabled(lc) == 0) {
257		RTE_LOG(ERR, USER1, "lcore %u is not enabled\n", lc);
258		return -EINVAL;
259	}
260	if (rte_eal_get_lcore_state(lc) == RUNNING) {
261		RTE_LOG(ERR, USER1, "lcore %u already running %p\n",
262			lc, lcore_config[lc].f);
263		return -EINVAL;
264	}
265	return 0;
266}
267
268static void
269log_netbe_prt(const struct netbe_port *uprt)
270{
271	uint32_t i;
272	char corelist[2 * RTE_MAX_LCORE + 1];
273	char hashkey[2 * RSS_HASH_KEY_LENGTH];
274
275	memset(corelist, 0, sizeof(corelist));
276	memset(hashkey, 0, sizeof(hashkey));
277	for (i = 0; i < uprt->nb_lcore; i++)
278		if (i < uprt->nb_lcore - 1)
279			sprintf(corelist + (2 * i), "%u,", uprt->lcore_id[i]);
280		else
281			sprintf(corelist + (2 * i), "%u", uprt->lcore_id[i]);
282
283	for (i = 0; i < uprt->hash_key_size; i++)
284		sprintf(hashkey + (2 * i), "%02x", uprt->hash_key[i]);
285
286	RTE_LOG(NOTICE, USER1,
287		"uprt %p = <id = %u, lcore = <%s>, mtu = %u, "
288		"rx_offload = %#" PRIx64 ", tx_offload = %#" PRIx64 ",\n"
289		"ipv4 = %#x, "
290		"ipv6 = %04hx:%04hx:%04hx:%04hx:%04hx:%04hx:%04hx:%04hx, "
291		"mac = %02hhx:%02hhx:%02hhx:%02hhx:%02hhx:%02hhx>;\n"
292		"hashkey = %s;\n",
293		uprt, uprt->id, corelist,
294		uprt->mtu, uprt->rx_offload, uprt->tx_offload,
295		uprt->ipv4,
296		uprt->ipv6.s6_addr16[0], uprt->ipv6.s6_addr16[1],
297		uprt->ipv6.s6_addr16[2], uprt->ipv6.s6_addr16[3],
298		uprt->ipv6.s6_addr16[4], uprt->ipv6.s6_addr16[5],
299		uprt->ipv6.s6_addr16[6], uprt->ipv6.s6_addr16[7],
300		uprt->mac.addr_bytes[0], uprt->mac.addr_bytes[1],
301		uprt->mac.addr_bytes[2], uprt->mac.addr_bytes[3],
302		uprt->mac.addr_bytes[4], uprt->mac.addr_bytes[5],
303		hashkey);
304}
305
306static void
307log_netbe_cfg(const struct netbe_cfg *ucfg)
308{
309	uint32_t i;
310
311	RTE_LOG(NOTICE, USER1,
312		"ucfg @ %p, prt_num = %u\n", ucfg, ucfg->prt_num);
313
314	for (i = 0; i != ucfg->prt_num; i++)
315		log_netbe_prt(ucfg->prt + i);
316}
317
318static int
319pool_init(uint32_t sid, uint32_t mpool_buf_num)
320{
321	int32_t rc;
322	struct rte_mempool *mp;
323	char name[RTE_MEMPOOL_NAMESIZE];
324
325	snprintf(name, sizeof(name), "MP%u", sid);
326	mp = rte_pktmbuf_pool_create(name, mpool_buf_num, MPOOL_CACHE_SIZE, 0,
327		RTE_MBUF_DEFAULT_BUF_SIZE, sid - 1);
328	if (mp == NULL) {
329		rc = -rte_errno;
330		RTE_LOG(ERR, USER1, "%s(%d) failed with error code: %d\n",
331			__func__, sid - 1, rc);
332		return rc;
333	}
334
335	mpool[sid] = mp;
336	return 0;
337}
338
339static int
340frag_pool_init(uint32_t sid, uint32_t mpool_buf_num)
341{
342	int32_t rc;
343	struct rte_mempool *frag_mp;
344	char frag_name[RTE_MEMPOOL_NAMESIZE];
345
346	snprintf(frag_name, sizeof(frag_name), "frag_MP%u", sid);
347	frag_mp = rte_pktmbuf_pool_create(frag_name, mpool_buf_num,
348		MPOOL_CACHE_SIZE, 0, FRAG_MBUF_BUF_SIZE, sid - 1);
349	if (frag_mp == NULL) {
350		rc = -rte_errno;
351		RTE_LOG(ERR, USER1, "%s(%d) failed with error code: %d\n",
352			__func__, sid - 1, rc);
353		return rc;
354	}
355
356	frag_mpool[sid] = frag_mp;
357	return 0;
358}
359
360static struct netbe_lcore *
361find_initilized_lcore(struct netbe_cfg *cfg, uint32_t lc_num)
362{
363	uint32_t i;
364
365	for (i = 0; i < cfg->cpu_num; i++)
366		if (cfg->cpu[i].id == lc_num)
367			return &cfg->cpu[i];
368
369	return NULL;
370}
371
372/*
373 * Setup all enabled ports.
374 */
375static int
376netbe_port_init(struct netbe_cfg *cfg)
377{
378	int32_t rc;
379	uint32_t i, sid, j;
380	struct netbe_port *prt;
381	struct netbe_lcore *lc;
382
383	for (i = 0; i != cfg->prt_num; i++) {
384		prt = cfg->prt + i;
385		rc = port_init(prt, cfg->proto);
386		if (rc != 0) {
387			RTE_LOG(ERR, USER1,
388				"%s: port=%u init failed with error code: %d\n",
389				__func__, prt->id, rc);
390			return rc;
391		}
392		rte_eth_macaddr_get(prt->id, &prt->mac);
393		if (cfg->promisc)
394			rte_eth_promiscuous_enable(prt->id);
395
396		for (j = 0; j < prt->nb_lcore; j++) {
397			rc = check_lcore(prt->lcore_id[j]);
398			if (rc != 0) {
399				RTE_LOG(ERR, USER1,
400					"%s: processing failed with err: %d\n",
401					__func__, rc);
402				return rc;
403			}
404
405			sid = rte_lcore_to_socket_id(prt->lcore_id[j]) + 1;
406			assert(sid < RTE_DIM(mpool));
407
408			if (mpool[sid] == NULL) {
409				rc = pool_init(sid, cfg->mpool_buf_num);
410				if (rc != 0)
411					return rc;
412			}
413
414			if (frag_mpool[sid] == NULL) {
415				rc = frag_pool_init(sid, cfg->mpool_buf_num);
416				if (rc != 0)
417					return rc;
418			}
419
420			rc = queue_init(prt, mpool[sid]);
421			if (rc != 0) {
422				RTE_LOG(ERR, USER1,
423					"%s: lcore=%u queue init failed with "
424					"err: %d\n",
425					__func__, prt->lcore_id[j], rc);
426				return rc;
427			}
428
429			/* calculate number of queues and assign queue id
430			 * per lcore. */
431			lc = find_initilized_lcore(cfg, prt->lcore_id[j]);
432			if (lc == NULL) {
433				lc = &cfg->cpu[cfg->cpu_num];
434				lc->id = prt->lcore_id[j];
435				lc->proto = becfg.proto;
436				cfg->cpu_num++;
437			}
438
439			lc->prtq = rte_realloc(lc->prtq, sizeof(*(lc->prtq)) *
440				(lc->prtq_num + 1), RTE_CACHE_LINE_SIZE);
441			if (lc->prtq == NULL) {
442				RTE_LOG(ERR, USER1,
443					"%s: failed to reallocate memory\n",
444					__func__);
445				return -ENOMEM;
446			}
447			lc->prtq[lc->prtq_num].rxqid = j;
448			lc->prtq[lc->prtq_num].txqid = j;
449			lc->prtq[lc->prtq_num].port = *prt;
450			lc->prtq_num++;
451		}
452	}
453	log_netbe_cfg(cfg);
454
455	return 0;
456}
457
458#endif /* PORT_H_ */
459