main.c revision fdd2322b
1/*-
2 *   BSD LICENSE
3 *
4 *   Copyright(c) 2015-2016 Intel Corporation. All rights reserved.
5 *   All rights reserved.
6 *
7 *   Redistribution and use in source and binary forms, with or without
8 *   modification, are permitted provided that the following conditions
9 *   are met:
10 *
11 *     * Redistributions of source code must retain the above copyright
12 *       notice, this list of conditions and the following disclaimer.
13 *     * Redistributions in binary form must reproduce the above copyright
14 *       notice, this list of conditions and the following disclaimer in
15 *       the documentation and/or other materials provided with the
16 *       distribution.
17 *     * Neither the name of Intel Corporation nor the names of its
18 *       contributors may be used to endorse or promote products derived
19 *       from this software without specific prior written permission.
20 *
21 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34#include <time.h>
35#include <stdio.h>
36#include <stdlib.h>
37#include <string.h>
38#include <stdint.h>
39#include <inttypes.h>
40#include <sys/types.h>
41#include <sys/queue.h>
42#include <netinet/in.h>
43#include <setjmp.h>
44#include <stdarg.h>
45#include <ctype.h>
46#include <errno.h>
47#include <getopt.h>
48#include <fcntl.h>
49#include <unistd.h>
50
51#include <rte_atomic.h>
52#include <rte_branch_prediction.h>
53#include <rte_common.h>
54#include <rte_cryptodev.h>
55#include <rte_cycles.h>
56#include <rte_debug.h>
57#include <rte_eal.h>
58#include <rte_ether.h>
59#include <rte_ethdev.h>
60#include <rte_interrupts.h>
61#include <rte_ip.h>
62#include <rte_launch.h>
63#include <rte_lcore.h>
64#include <rte_log.h>
65#include <rte_malloc.h>
66#include <rte_mbuf.h>
67#include <rte_memcpy.h>
68#include <rte_memory.h>
69#include <rte_mempool.h>
70#include <rte_memzone.h>
71#include <rte_pci.h>
72#include <rte_per_lcore.h>
73#include <rte_prefetch.h>
74#include <rte_random.h>
75#include <rte_hexdump.h>
76
77enum cdev_type {
78	CDEV_TYPE_ANY,
79	CDEV_TYPE_HW,
80	CDEV_TYPE_SW
81};
82
83#define RTE_LOGTYPE_L2FWD RTE_LOGTYPE_USER1
84
85#define NB_MBUF   8192
86
87#define MAX_STR_LEN 32
88#define MAX_KEY_SIZE 128
89#define MAX_PKT_BURST 32
90#define BURST_TX_DRAIN_US 100 /* TX drain every ~100us */
91
92/*
93 * Configurable number of RX/TX ring descriptors
94 */
95#define RTE_TEST_RX_DESC_DEFAULT 128
96#define RTE_TEST_TX_DESC_DEFAULT 512
97
98static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT;
99static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT;
100
101/* ethernet addresses of ports */
102static struct ether_addr l2fwd_ports_eth_addr[RTE_MAX_ETHPORTS];
103
104/* mask of enabled ports */
105static uint64_t l2fwd_enabled_port_mask;
106static uint64_t l2fwd_enabled_crypto_mask;
107
108/* list of enabled ports */
109static uint32_t l2fwd_dst_ports[RTE_MAX_ETHPORTS];
110
111
112struct pkt_buffer {
113	unsigned len;
114	struct rte_mbuf *buffer[MAX_PKT_BURST];
115};
116
117struct op_buffer {
118	unsigned len;
119	struct rte_crypto_op *buffer[MAX_PKT_BURST];
120};
121
122#define MAX_RX_QUEUE_PER_LCORE 16
123#define MAX_TX_QUEUE_PER_PORT 16
124
125enum l2fwd_crypto_xform_chain {
126	L2FWD_CRYPTO_CIPHER_HASH,
127	L2FWD_CRYPTO_HASH_CIPHER,
128	L2FWD_CRYPTO_CIPHER_ONLY,
129	L2FWD_CRYPTO_HASH_ONLY
130};
131
132struct l2fwd_key {
133	uint8_t *data;
134	uint32_t length;
135	phys_addr_t phys_addr;
136};
137
138char supported_auth_algo[RTE_CRYPTO_AUTH_LIST_END][MAX_STR_LEN];
139char supported_cipher_algo[RTE_CRYPTO_CIPHER_LIST_END][MAX_STR_LEN];
140
141/** l2fwd crypto application command line options */
142struct l2fwd_crypto_options {
143	unsigned portmask;
144	unsigned nb_ports_per_lcore;
145	unsigned refresh_period;
146	unsigned single_lcore:1;
147
148	enum cdev_type type;
149	unsigned sessionless:1;
150
151	enum l2fwd_crypto_xform_chain xform_chain;
152
153	struct rte_crypto_sym_xform cipher_xform;
154	unsigned ckey_param;
155	int ckey_random_size;
156
157	struct l2fwd_key iv;
158	unsigned iv_param;
159	int iv_random_size;
160
161	struct rte_crypto_sym_xform auth_xform;
162	uint8_t akey_param;
163	int akey_random_size;
164
165	struct l2fwd_key aad;
166	unsigned aad_param;
167	int aad_random_size;
168
169	int digest_size;
170
171	uint16_t block_size;
172	char string_type[MAX_STR_LEN];
173};
174
175/** l2fwd crypto lcore params */
176struct l2fwd_crypto_params {
177	uint8_t dev_id;
178	uint8_t qp_id;
179
180	unsigned digest_length;
181	unsigned block_size;
182
183	struct l2fwd_key iv;
184	struct l2fwd_key aad;
185	struct rte_cryptodev_sym_session *session;
186
187	uint8_t do_cipher;
188	uint8_t do_hash;
189	uint8_t hash_verify;
190
191	enum rte_crypto_cipher_algorithm cipher_algo;
192	enum rte_crypto_auth_algorithm auth_algo;
193};
194
195/** lcore configuration */
196struct lcore_queue_conf {
197	unsigned nb_rx_ports;
198	unsigned rx_port_list[MAX_RX_QUEUE_PER_LCORE];
199
200	unsigned nb_crypto_devs;
201	unsigned cryptodev_list[MAX_RX_QUEUE_PER_LCORE];
202
203	struct op_buffer op_buf[RTE_CRYPTO_MAX_DEVS];
204	struct pkt_buffer pkt_buf[RTE_MAX_ETHPORTS];
205} __rte_cache_aligned;
206
207struct lcore_queue_conf lcore_queue_conf[RTE_MAX_LCORE];
208
209static const struct rte_eth_conf port_conf = {
210	.rxmode = {
211		.mq_mode = ETH_MQ_RX_NONE,
212		.max_rx_pkt_len = ETHER_MAX_LEN,
213		.split_hdr_size = 0,
214		.header_split   = 0, /**< Header Split disabled */
215		.hw_ip_checksum = 0, /**< IP checksum offload disabled */
216		.hw_vlan_filter = 0, /**< VLAN filtering disabled */
217		.jumbo_frame    = 0, /**< Jumbo Frame Support disabled */
218		.hw_strip_crc   = 1, /**< CRC stripped by hardware */
219	},
220	.txmode = {
221		.mq_mode = ETH_MQ_TX_NONE,
222	},
223};
224
225struct rte_mempool *l2fwd_pktmbuf_pool;
226struct rte_mempool *l2fwd_crypto_op_pool;
227
228/* Per-port statistics struct */
229struct l2fwd_port_statistics {
230	uint64_t tx;
231	uint64_t rx;
232
233	uint64_t crypto_enqueued;
234	uint64_t crypto_dequeued;
235
236	uint64_t dropped;
237} __rte_cache_aligned;
238
239struct l2fwd_crypto_statistics {
240	uint64_t enqueued;
241	uint64_t dequeued;
242
243	uint64_t errors;
244} __rte_cache_aligned;
245
246struct l2fwd_port_statistics port_statistics[RTE_MAX_ETHPORTS];
247struct l2fwd_crypto_statistics crypto_statistics[RTE_CRYPTO_MAX_DEVS];
248
249/* A tsc-based timer responsible for triggering statistics printout */
250#define TIMER_MILLISECOND 2000000ULL /* around 1ms at 2 Ghz */
251#define MAX_TIMER_PERIOD 86400UL /* 1 day max */
252
253/* default period is 10 seconds */
254static int64_t timer_period = 10 * TIMER_MILLISECOND * 1000;
255
256/* Print out statistics on packets dropped */
257static void
258print_stats(void)
259{
260	uint64_t total_packets_dropped, total_packets_tx, total_packets_rx;
261	uint64_t total_packets_enqueued, total_packets_dequeued,
262		total_packets_errors;
263	unsigned portid;
264	uint64_t cdevid;
265
266	total_packets_dropped = 0;
267	total_packets_tx = 0;
268	total_packets_rx = 0;
269	total_packets_enqueued = 0;
270	total_packets_dequeued = 0;
271	total_packets_errors = 0;
272
273	const char clr[] = { 27, '[', '2', 'J', '\0' };
274	const char topLeft[] = { 27, '[', '1', ';', '1', 'H', '\0' };
275
276		/* Clear screen and move to top left */
277	printf("%s%s", clr, topLeft);
278
279	printf("\nPort statistics ====================================");
280
281	for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) {
282		/* skip disabled ports */
283		if ((l2fwd_enabled_port_mask & (1 << portid)) == 0)
284			continue;
285		printf("\nStatistics for port %u ------------------------------"
286			   "\nPackets sent: %32"PRIu64
287			   "\nPackets received: %28"PRIu64
288			   "\nPackets dropped: %29"PRIu64,
289			   portid,
290			   port_statistics[portid].tx,
291			   port_statistics[portid].rx,
292			   port_statistics[portid].dropped);
293
294		total_packets_dropped += port_statistics[portid].dropped;
295		total_packets_tx += port_statistics[portid].tx;
296		total_packets_rx += port_statistics[portid].rx;
297	}
298	printf("\nCrypto statistics ==================================");
299
300	for (cdevid = 0; cdevid < RTE_CRYPTO_MAX_DEVS; cdevid++) {
301		/* skip disabled ports */
302		if ((l2fwd_enabled_crypto_mask & (((uint64_t)1) << cdevid)) == 0)
303			continue;
304		printf("\nStatistics for cryptodev %"PRIu64
305				" -------------------------"
306			   "\nPackets enqueued: %28"PRIu64
307			   "\nPackets dequeued: %28"PRIu64
308			   "\nPackets errors: %30"PRIu64,
309			   cdevid,
310			   crypto_statistics[cdevid].enqueued,
311			   crypto_statistics[cdevid].dequeued,
312			   crypto_statistics[cdevid].errors);
313
314		total_packets_enqueued += crypto_statistics[cdevid].enqueued;
315		total_packets_dequeued += crypto_statistics[cdevid].dequeued;
316		total_packets_errors += crypto_statistics[cdevid].errors;
317	}
318	printf("\nAggregate statistics ==============================="
319		   "\nTotal packets received: %22"PRIu64
320		   "\nTotal packets enqueued: %22"PRIu64
321		   "\nTotal packets dequeued: %22"PRIu64
322		   "\nTotal packets sent: %26"PRIu64
323		   "\nTotal packets dropped: %23"PRIu64
324		   "\nTotal packets crypto errors: %17"PRIu64,
325		   total_packets_rx,
326		   total_packets_enqueued,
327		   total_packets_dequeued,
328		   total_packets_tx,
329		   total_packets_dropped,
330		   total_packets_errors);
331	printf("\n====================================================\n");
332}
333
334static void
335fill_supported_algorithm_tables(void)
336{
337	unsigned i;
338
339	for (i = 0; i < RTE_CRYPTO_AUTH_LIST_END; i++)
340		strcpy(supported_auth_algo[i], "NOT_SUPPORTED");
341
342	strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_AES_GCM], "AES_GCM");
343	strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_AES_GMAC], "AES_GMAC");
344	strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_MD5_HMAC], "MD5_HMAC");
345	strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_MD5], "MD5");
346	strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_NULL], "NULL");
347	strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_AES_XCBC_MAC],
348		"AES_XCBC_MAC");
349	strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_SHA1_HMAC], "SHA1_HMAC");
350	strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_SHA1], "SHA1");
351	strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_SHA224_HMAC], "SHA224_HMAC");
352	strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_SHA224], "SHA224");
353	strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_SHA256_HMAC], "SHA256_HMAC");
354	strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_SHA256], "SHA256");
355	strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_SHA384_HMAC], "SHA384_HMAC");
356	strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_SHA384], "SHA384");
357	strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_SHA512_HMAC], "SHA512_HMAC");
358	strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_SHA512], "SHA512");
359	strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_SNOW3G_UIA2], "SNOW3G_UIA2");
360	strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_ZUC_EIA3], "ZUC_EIA3");
361	strcpy(supported_auth_algo[RTE_CRYPTO_AUTH_KASUMI_F9], "KASUMI_F9");
362
363	for (i = 0; i < RTE_CRYPTO_CIPHER_LIST_END; i++)
364		strcpy(supported_cipher_algo[i], "NOT_SUPPORTED");
365
366	strcpy(supported_cipher_algo[RTE_CRYPTO_CIPHER_AES_CBC], "AES_CBC");
367	strcpy(supported_cipher_algo[RTE_CRYPTO_CIPHER_AES_CTR], "AES_CTR");
368	strcpy(supported_cipher_algo[RTE_CRYPTO_CIPHER_AES_GCM], "AES_GCM");
369	strcpy(supported_cipher_algo[RTE_CRYPTO_CIPHER_NULL], "NULL");
370	strcpy(supported_cipher_algo[RTE_CRYPTO_CIPHER_SNOW3G_UEA2], "SNOW3G_UEA2");
371	strcpy(supported_cipher_algo[RTE_CRYPTO_CIPHER_ZUC_EEA3], "ZUC_EEA3");
372	strcpy(supported_cipher_algo[RTE_CRYPTO_CIPHER_KASUMI_F8], "KASUMI_F8");
373	strcpy(supported_cipher_algo[RTE_CRYPTO_CIPHER_3DES_CTR], "3DES_CTR");
374	strcpy(supported_cipher_algo[RTE_CRYPTO_CIPHER_3DES_CBC], "3DES_CBC");
375}
376
377
378static int
379l2fwd_crypto_send_burst(struct lcore_queue_conf *qconf, unsigned n,
380		struct l2fwd_crypto_params *cparams)
381{
382	struct rte_crypto_op **op_buffer;
383	unsigned ret;
384
385	op_buffer = (struct rte_crypto_op **)
386			qconf->op_buf[cparams->dev_id].buffer;
387
388	ret = rte_cryptodev_enqueue_burst(cparams->dev_id,
389			cparams->qp_id,	op_buffer, (uint16_t) n);
390
391	crypto_statistics[cparams->dev_id].enqueued += ret;
392	if (unlikely(ret < n)) {
393		crypto_statistics[cparams->dev_id].errors += (n - ret);
394		do {
395			rte_pktmbuf_free(op_buffer[ret]->sym->m_src);
396			rte_crypto_op_free(op_buffer[ret]);
397		} while (++ret < n);
398	}
399
400	return 0;
401}
402
403static int
404l2fwd_crypto_enqueue(struct rte_crypto_op *op,
405		struct l2fwd_crypto_params *cparams)
406{
407	unsigned lcore_id, len;
408	struct lcore_queue_conf *qconf;
409
410	lcore_id = rte_lcore_id();
411
412	qconf = &lcore_queue_conf[lcore_id];
413	len = qconf->op_buf[cparams->dev_id].len;
414	qconf->op_buf[cparams->dev_id].buffer[len] = op;
415	len++;
416
417	/* enough ops to be sent */
418	if (len == MAX_PKT_BURST) {
419		l2fwd_crypto_send_burst(qconf, MAX_PKT_BURST, cparams);
420		len = 0;
421	}
422
423	qconf->op_buf[cparams->dev_id].len = len;
424	return 0;
425}
426
427static int
428l2fwd_simple_crypto_enqueue(struct rte_mbuf *m,
429		struct rte_crypto_op *op,
430		struct l2fwd_crypto_params *cparams)
431{
432	struct ether_hdr *eth_hdr;
433	struct ipv4_hdr *ip_hdr;
434
435	uint32_t ipdata_offset, data_len;
436	uint32_t pad_len = 0;
437	char *padding;
438
439	eth_hdr = rte_pktmbuf_mtod(m, struct ether_hdr *);
440
441	if (eth_hdr->ether_type != rte_cpu_to_be_16(ETHER_TYPE_IPv4))
442		return -1;
443
444	ipdata_offset = sizeof(struct ether_hdr);
445
446	ip_hdr = (struct ipv4_hdr *)(rte_pktmbuf_mtod(m, char *) +
447			ipdata_offset);
448
449	ipdata_offset += (ip_hdr->version_ihl & IPV4_HDR_IHL_MASK)
450			* IPV4_IHL_MULTIPLIER;
451
452
453	/* Zero pad data to be crypto'd so it is block aligned */
454	data_len  = rte_pktmbuf_data_len(m) - ipdata_offset;
455
456	if (cparams->do_hash && cparams->hash_verify)
457		data_len -= cparams->digest_length;
458
459	if (cparams->do_cipher) {
460		/*
461		 * Following algorithms are block cipher algorithms,
462		 * and might need padding
463		 */
464		switch (cparams->cipher_algo) {
465		case RTE_CRYPTO_CIPHER_AES_CBC:
466		case RTE_CRYPTO_CIPHER_AES_ECB:
467		case RTE_CRYPTO_CIPHER_3DES_CBC:
468		case RTE_CRYPTO_CIPHER_3DES_ECB:
469			if (data_len % cparams->block_size)
470				pad_len = cparams->block_size -
471					(data_len % cparams->block_size);
472			break;
473		default:
474			pad_len = 0;
475		}
476
477		if (pad_len) {
478			padding = rte_pktmbuf_append(m, pad_len);
479			if (unlikely(!padding))
480				return -1;
481
482			data_len += pad_len;
483			memset(padding, 0, pad_len);
484		}
485	}
486
487	/* Set crypto operation data parameters */
488	rte_crypto_op_attach_sym_session(op, cparams->session);
489
490	if (cparams->do_hash) {
491		if (!cparams->hash_verify) {
492			/* Append space for digest to end of packet */
493			op->sym->auth.digest.data = (uint8_t *)rte_pktmbuf_append(m,
494				cparams->digest_length);
495		} else {
496			op->sym->auth.digest.data = rte_pktmbuf_mtod(m,
497				uint8_t *) + ipdata_offset + data_len;
498		}
499
500		op->sym->auth.digest.phys_addr = rte_pktmbuf_mtophys_offset(m,
501				rte_pktmbuf_pkt_len(m) - cparams->digest_length);
502		op->sym->auth.digest.length = cparams->digest_length;
503
504		/* For wireless algorithms, offset/length must be in bits */
505		if (cparams->auth_algo == RTE_CRYPTO_AUTH_SNOW3G_UIA2 ||
506				cparams->auth_algo == RTE_CRYPTO_AUTH_KASUMI_F9 ||
507				cparams->auth_algo == RTE_CRYPTO_AUTH_ZUC_EIA3) {
508			op->sym->auth.data.offset = ipdata_offset << 3;
509			op->sym->auth.data.length = data_len << 3;
510		} else {
511			op->sym->auth.data.offset = ipdata_offset;
512			op->sym->auth.data.length = data_len;
513		}
514
515		if (cparams->aad.length) {
516			op->sym->auth.aad.data = cparams->aad.data;
517			op->sym->auth.aad.phys_addr = cparams->aad.phys_addr;
518			op->sym->auth.aad.length = cparams->aad.length;
519		} else {
520			op->sym->auth.aad.data = NULL;
521			op->sym->auth.aad.phys_addr = 0;
522			op->sym->auth.aad.length = 0;
523		}
524	}
525
526	if (cparams->do_cipher) {
527		op->sym->cipher.iv.data = cparams->iv.data;
528		op->sym->cipher.iv.phys_addr = cparams->iv.phys_addr;
529		op->sym->cipher.iv.length = cparams->iv.length;
530
531		/* For wireless algorithms, offset/length must be in bits */
532		if (cparams->cipher_algo == RTE_CRYPTO_CIPHER_SNOW3G_UEA2 ||
533				cparams->cipher_algo == RTE_CRYPTO_CIPHER_KASUMI_F8 ||
534				cparams->cipher_algo == RTE_CRYPTO_CIPHER_ZUC_EEA3) {
535			op->sym->cipher.data.offset = ipdata_offset << 3;
536			op->sym->cipher.data.length = data_len << 3;
537		} else {
538			op->sym->cipher.data.offset = ipdata_offset;
539			op->sym->cipher.data.length = data_len;
540		}
541	}
542
543	op->sym->m_src = m;
544
545	return l2fwd_crypto_enqueue(op, cparams);
546}
547
548
549/* Send the burst of packets on an output interface */
550static int
551l2fwd_send_burst(struct lcore_queue_conf *qconf, unsigned n,
552		uint8_t port)
553{
554	struct rte_mbuf **pkt_buffer;
555	unsigned ret;
556
557	pkt_buffer = (struct rte_mbuf **)qconf->pkt_buf[port].buffer;
558
559	ret = rte_eth_tx_burst(port, 0, pkt_buffer, (uint16_t)n);
560	port_statistics[port].tx += ret;
561	if (unlikely(ret < n)) {
562		port_statistics[port].dropped += (n - ret);
563		do {
564			rte_pktmbuf_free(pkt_buffer[ret]);
565		} while (++ret < n);
566	}
567
568	return 0;
569}
570
571/* Enqueue packets for TX and prepare them to be sent */
572static int
573l2fwd_send_packet(struct rte_mbuf *m, uint8_t port)
574{
575	unsigned lcore_id, len;
576	struct lcore_queue_conf *qconf;
577
578	lcore_id = rte_lcore_id();
579
580	qconf = &lcore_queue_conf[lcore_id];
581	len = qconf->pkt_buf[port].len;
582	qconf->pkt_buf[port].buffer[len] = m;
583	len++;
584
585	/* enough pkts to be sent */
586	if (unlikely(len == MAX_PKT_BURST)) {
587		l2fwd_send_burst(qconf, MAX_PKT_BURST, port);
588		len = 0;
589	}
590
591	qconf->pkt_buf[port].len = len;
592	return 0;
593}
594
595static void
596l2fwd_simple_forward(struct rte_mbuf *m, unsigned portid)
597{
598	struct ether_hdr *eth;
599	void *tmp;
600	unsigned dst_port;
601
602	dst_port = l2fwd_dst_ports[portid];
603	eth = rte_pktmbuf_mtod(m, struct ether_hdr *);
604
605	/* 02:00:00:00:00:xx */
606	tmp = &eth->d_addr.addr_bytes[0];
607	*((uint64_t *)tmp) = 0x000000000002 + ((uint64_t)dst_port << 40);
608
609	/* src addr */
610	ether_addr_copy(&l2fwd_ports_eth_addr[dst_port], &eth->s_addr);
611
612	l2fwd_send_packet(m, (uint8_t) dst_port);
613}
614
615/** Generate random key */
616static void
617generate_random_key(uint8_t *key, unsigned length)
618{
619	int fd;
620	int ret;
621
622	fd = open("/dev/urandom", O_RDONLY);
623	if (fd < 0)
624		rte_exit(EXIT_FAILURE, "Failed to generate random key\n");
625
626	ret = read(fd, key, length);
627	close(fd);
628
629	if (ret != (signed)length)
630		rte_exit(EXIT_FAILURE, "Failed to generate random key\n");
631}
632
633static struct rte_cryptodev_sym_session *
634initialize_crypto_session(struct l2fwd_crypto_options *options,
635		uint8_t cdev_id)
636{
637	struct rte_crypto_sym_xform *first_xform;
638
639	if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH) {
640		first_xform = &options->cipher_xform;
641		first_xform->next = &options->auth_xform;
642	} else if (options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER) {
643		first_xform = &options->auth_xform;
644		first_xform->next = &options->cipher_xform;
645	} else if (options->xform_chain == L2FWD_CRYPTO_CIPHER_ONLY) {
646		first_xform = &options->cipher_xform;
647	} else {
648		first_xform = &options->auth_xform;
649	}
650
651	/* Setup Cipher Parameters */
652	return rte_cryptodev_sym_session_create(cdev_id, first_xform);
653}
654
655static void
656l2fwd_crypto_options_print(struct l2fwd_crypto_options *options);
657
658/* main processing loop */
659static void
660l2fwd_main_loop(struct l2fwd_crypto_options *options)
661{
662	struct rte_mbuf *m, *pkts_burst[MAX_PKT_BURST];
663	struct rte_crypto_op *ops_burst[MAX_PKT_BURST];
664
665	unsigned lcore_id = rte_lcore_id();
666	uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0;
667	unsigned i, j, portid, nb_rx, len;
668	struct lcore_queue_conf *qconf = &lcore_queue_conf[lcore_id];
669	const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1) /
670			US_PER_S * BURST_TX_DRAIN_US;
671	struct l2fwd_crypto_params *cparams;
672	struct l2fwd_crypto_params port_cparams[qconf->nb_crypto_devs];
673
674	if (qconf->nb_rx_ports == 0) {
675		RTE_LOG(INFO, L2FWD, "lcore %u has nothing to do\n", lcore_id);
676		return;
677	}
678
679	RTE_LOG(INFO, L2FWD, "entering main loop on lcore %u\n", lcore_id);
680
681	for (i = 0; i < qconf->nb_rx_ports; i++) {
682
683		portid = qconf->rx_port_list[i];
684		RTE_LOG(INFO, L2FWD, " -- lcoreid=%u portid=%u\n", lcore_id,
685			portid);
686	}
687
688	for (i = 0; i < qconf->nb_crypto_devs; i++) {
689		port_cparams[i].do_cipher = 0;
690		port_cparams[i].do_hash = 0;
691
692		switch (options->xform_chain) {
693		case L2FWD_CRYPTO_CIPHER_HASH:
694		case L2FWD_CRYPTO_HASH_CIPHER:
695			port_cparams[i].do_cipher = 1;
696			port_cparams[i].do_hash = 1;
697			break;
698		case L2FWD_CRYPTO_HASH_ONLY:
699			port_cparams[i].do_hash = 1;
700			break;
701		case L2FWD_CRYPTO_CIPHER_ONLY:
702			port_cparams[i].do_cipher = 1;
703			break;
704		}
705
706		port_cparams[i].dev_id = qconf->cryptodev_list[i];
707		port_cparams[i].qp_id = 0;
708
709		port_cparams[i].block_size = options->block_size;
710
711		if (port_cparams[i].do_hash) {
712			port_cparams[i].digest_length =
713					options->auth_xform.auth.digest_length;
714			if (options->auth_xform.auth.add_auth_data_length) {
715				port_cparams[i].aad.data = options->aad.data;
716				port_cparams[i].aad.length =
717					options->auth_xform.auth.add_auth_data_length;
718				port_cparams[i].aad.phys_addr = options->aad.phys_addr;
719				if (!options->aad_param)
720					generate_random_key(port_cparams[i].aad.data,
721						port_cparams[i].aad.length);
722
723			} else
724				port_cparams[i].aad.length = 0;
725
726			if (options->auth_xform.auth.op == RTE_CRYPTO_AUTH_OP_VERIFY)
727				port_cparams[i].hash_verify = 1;
728			else
729				port_cparams[i].hash_verify = 0;
730
731			port_cparams[i].auth_algo = options->auth_xform.auth.algo;
732		}
733
734		if (port_cparams[i].do_cipher) {
735			port_cparams[i].iv.data = options->iv.data;
736			port_cparams[i].iv.length = options->iv.length;
737			port_cparams[i].iv.phys_addr = options->iv.phys_addr;
738			if (!options->iv_param)
739				generate_random_key(port_cparams[i].iv.data,
740						port_cparams[i].iv.length);
741
742			port_cparams[i].cipher_algo = options->cipher_xform.cipher.algo;
743		}
744
745		port_cparams[i].session = initialize_crypto_session(options,
746				port_cparams[i].dev_id);
747
748		if (port_cparams[i].session == NULL)
749			return;
750		RTE_LOG(INFO, L2FWD, " -- lcoreid=%u cryptoid=%u\n", lcore_id,
751				port_cparams[i].dev_id);
752	}
753
754	l2fwd_crypto_options_print(options);
755
756	/*
757	 * Initialize previous tsc timestamp before the loop,
758	 * to avoid showing the port statistics immediately,
759	 * so user can see the crypto information.
760	 */
761	prev_tsc = rte_rdtsc();
762	while (1) {
763
764		cur_tsc = rte_rdtsc();
765
766		/*
767		 * Crypto device/TX burst queue drain
768		 */
769		diff_tsc = cur_tsc - prev_tsc;
770		if (unlikely(diff_tsc > drain_tsc)) {
771			/* Enqueue all crypto ops remaining in buffers */
772			for (i = 0; i < qconf->nb_crypto_devs; i++) {
773				cparams = &port_cparams[i];
774				len = qconf->op_buf[cparams->dev_id].len;
775				l2fwd_crypto_send_burst(qconf, len, cparams);
776				qconf->op_buf[cparams->dev_id].len = 0;
777			}
778			/* Transmit all packets remaining in buffers */
779			for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++) {
780				if (qconf->pkt_buf[portid].len == 0)
781					continue;
782				l2fwd_send_burst(&lcore_queue_conf[lcore_id],
783						 qconf->pkt_buf[portid].len,
784						 (uint8_t) portid);
785				qconf->pkt_buf[portid].len = 0;
786			}
787
788			/* if timer is enabled */
789			if (timer_period > 0) {
790
791				/* advance the timer */
792				timer_tsc += diff_tsc;
793
794				/* if timer has reached its timeout */
795				if (unlikely(timer_tsc >=
796						(uint64_t)timer_period)) {
797
798					/* do this only on master core */
799					if (lcore_id == rte_get_master_lcore()
800						&& options->refresh_period) {
801						print_stats();
802						timer_tsc = 0;
803					}
804				}
805			}
806
807			prev_tsc = cur_tsc;
808		}
809
810		/*
811		 * Read packet from RX queues
812		 */
813		for (i = 0; i < qconf->nb_rx_ports; i++) {
814			portid = qconf->rx_port_list[i];
815
816			cparams = &port_cparams[i];
817
818			nb_rx = rte_eth_rx_burst((uint8_t) portid, 0,
819						 pkts_burst, MAX_PKT_BURST);
820
821			port_statistics[portid].rx += nb_rx;
822
823			if (nb_rx) {
824				/*
825				 * If we can't allocate a crypto_ops, then drop
826				 * the rest of the burst and dequeue and
827				 * process the packets to free offload structs
828				 */
829				if (rte_crypto_op_bulk_alloc(
830						l2fwd_crypto_op_pool,
831						RTE_CRYPTO_OP_TYPE_SYMMETRIC,
832						ops_burst, nb_rx) !=
833								nb_rx) {
834					for (j = 0; j < nb_rx; j++)
835						rte_pktmbuf_free(pkts_burst[j]);
836
837					nb_rx = 0;
838				}
839
840				/* Enqueue packets from Crypto device*/
841				for (j = 0; j < nb_rx; j++) {
842					m = pkts_burst[j];
843
844					l2fwd_simple_crypto_enqueue(m,
845							ops_burst[j], cparams);
846				}
847			}
848
849			/* Dequeue packets from Crypto device */
850			do {
851				nb_rx = rte_cryptodev_dequeue_burst(
852						cparams->dev_id, cparams->qp_id,
853						ops_burst, MAX_PKT_BURST);
854
855				crypto_statistics[cparams->dev_id].dequeued +=
856						nb_rx;
857
858				/* Forward crypto'd packets */
859				for (j = 0; j < nb_rx; j++) {
860					m = ops_burst[j]->sym->m_src;
861
862					rte_crypto_op_free(ops_burst[j]);
863					l2fwd_simple_forward(m, portid);
864				}
865			} while (nb_rx == MAX_PKT_BURST);
866		}
867	}
868}
869
870static int
871l2fwd_launch_one_lcore(void *arg)
872{
873	l2fwd_main_loop((struct l2fwd_crypto_options *)arg);
874	return 0;
875}
876
877/* Display command line arguments usage */
878static void
879l2fwd_crypto_usage(const char *prgname)
880{
881	printf("%s [EAL options] --\n"
882		"  -p PORTMASK: hexadecimal bitmask of ports to configure\n"
883		"  -q NQ: number of queue (=ports) per lcore (default is 1)\n"
884		"  -s manage all ports from single lcore\n"
885		"  -T PERIOD: statistics will be refreshed each PERIOD seconds"
886		" (0 to disable, 10 default, 86400 maximum)\n"
887
888		"  --cdev_type HW / SW / ANY\n"
889		"  --chain HASH_CIPHER / CIPHER_HASH / CIPHER_ONLY /"
890		" HASH_ONLY\n"
891
892		"  --cipher_algo ALGO\n"
893		"  --cipher_op ENCRYPT / DECRYPT\n"
894		"  --cipher_key KEY (bytes separated with \":\")\n"
895		"  --cipher_key_random_size SIZE: size of cipher key when generated randomly\n"
896		"  --iv IV (bytes separated with \":\")\n"
897		"  --iv_random_size SIZE: size of IV when generated randomly\n"
898
899		"  --auth_algo ALGO\n"
900		"  --auth_op GENERATE / VERIFY\n"
901		"  --auth_key KEY (bytes separated with \":\")\n"
902		"  --auth_key_random_size SIZE: size of auth key when generated randomly\n"
903		"  --aad AAD (bytes separated with \":\")\n"
904		"  --aad_random_size SIZE: size of AAD when generated randomly\n"
905		"  --digest_size SIZE: size of digest to be generated/verified\n"
906
907		"  --sessionless\n",
908	       prgname);
909}
910
911/** Parse crypto device type command line argument */
912static int
913parse_cryptodev_type(enum cdev_type *type, char *optarg)
914{
915	if (strcmp("HW", optarg) == 0) {
916		*type = CDEV_TYPE_HW;
917		return 0;
918	} else if (strcmp("SW", optarg) == 0) {
919		*type = CDEV_TYPE_SW;
920		return 0;
921	} else if (strcmp("ANY", optarg) == 0) {
922		*type = CDEV_TYPE_ANY;
923		return 0;
924	}
925
926	return -1;
927}
928
929/** Parse crypto chain xform command line argument */
930static int
931parse_crypto_opt_chain(struct l2fwd_crypto_options *options, char *optarg)
932{
933	if (strcmp("CIPHER_HASH", optarg) == 0) {
934		options->xform_chain = L2FWD_CRYPTO_CIPHER_HASH;
935		return 0;
936	} else if (strcmp("HASH_CIPHER", optarg) == 0) {
937		options->xform_chain = L2FWD_CRYPTO_HASH_CIPHER;
938		return 0;
939	} else if (strcmp("CIPHER_ONLY", optarg) == 0) {
940		options->xform_chain = L2FWD_CRYPTO_CIPHER_ONLY;
941		return 0;
942	} else if (strcmp("HASH_ONLY", optarg) == 0) {
943		options->xform_chain = L2FWD_CRYPTO_HASH_ONLY;
944		return 0;
945	}
946
947	return -1;
948}
949
950/** Parse crypto cipher algo option command line argument */
951static int
952parse_cipher_algo(enum rte_crypto_cipher_algorithm *algo, char *optarg)
953{
954	unsigned i;
955
956	for (i = 0; i < RTE_CRYPTO_CIPHER_LIST_END; i++) {
957		if (!strcmp(supported_cipher_algo[i], optarg)) {
958			*algo = (enum rte_crypto_cipher_algorithm)i;
959			return 0;
960		}
961	}
962
963	printf("Cipher algorithm  not supported!\n");
964	return -1;
965}
966
967/** Parse crypto cipher operation command line argument */
968static int
969parse_cipher_op(enum rte_crypto_cipher_operation *op, char *optarg)
970{
971	if (strcmp("ENCRYPT", optarg) == 0) {
972		*op = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
973		return 0;
974	} else if (strcmp("DECRYPT", optarg) == 0) {
975		*op = RTE_CRYPTO_CIPHER_OP_DECRYPT;
976		return 0;
977	}
978
979	printf("Cipher operation not supported!\n");
980	return -1;
981}
982
983/** Parse crypto key command line argument */
984static int
985parse_key(uint8_t *data, char *input_arg)
986{
987	unsigned byte_count;
988	char *token;
989
990	for (byte_count = 0, token = strtok(input_arg, ":");
991			(byte_count < MAX_KEY_SIZE) && (token != NULL);
992			token = strtok(NULL, ":")) {
993
994		int number = (int)strtol(token, NULL, 16);
995
996		if (errno == EINVAL || errno == ERANGE || number > 0xFF)
997			return -1;
998
999		data[byte_count++] = (uint8_t)number;
1000	}
1001
1002	return byte_count;
1003}
1004
1005/** Parse size param*/
1006static int
1007parse_size(int *size, const char *q_arg)
1008{
1009	char *end = NULL;
1010	unsigned long n;
1011
1012	/* parse hexadecimal string */
1013	n = strtoul(q_arg, &end, 10);
1014	if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0'))
1015		n = 0;
1016
1017	if (n == 0) {
1018		printf("invalid size\n");
1019		return -1;
1020	}
1021
1022	*size = n;
1023	return 0;
1024}
1025
1026/** Parse crypto cipher operation command line argument */
1027static int
1028parse_auth_algo(enum rte_crypto_auth_algorithm *algo, char *optarg)
1029{
1030	unsigned i;
1031
1032	for (i = 0; i < RTE_CRYPTO_AUTH_LIST_END; i++) {
1033		if (!strcmp(supported_auth_algo[i], optarg)) {
1034			*algo = (enum rte_crypto_auth_algorithm)i;
1035			return 0;
1036		}
1037	}
1038
1039	printf("Authentication algorithm specified not supported!\n");
1040	return -1;
1041}
1042
1043static int
1044parse_auth_op(enum rte_crypto_auth_operation *op, char *optarg)
1045{
1046	if (strcmp("VERIFY", optarg) == 0) {
1047		*op = RTE_CRYPTO_AUTH_OP_VERIFY;
1048		return 0;
1049	} else if (strcmp("GENERATE", optarg) == 0) {
1050		*op = RTE_CRYPTO_AUTH_OP_GENERATE;
1051		return 0;
1052	}
1053
1054	printf("Authentication operation specified not supported!\n");
1055	return -1;
1056}
1057
1058/** Parse long options */
1059static int
1060l2fwd_crypto_parse_args_long_options(struct l2fwd_crypto_options *options,
1061		struct option *lgopts, int option_index)
1062{
1063	int retval;
1064
1065	if (strcmp(lgopts[option_index].name, "cdev_type") == 0) {
1066		retval = parse_cryptodev_type(&options->type, optarg);
1067		if (retval == 0)
1068			snprintf(options->string_type, MAX_STR_LEN,
1069				"%s", optarg);
1070		return retval;
1071	}
1072
1073	else if (strcmp(lgopts[option_index].name, "chain") == 0)
1074		return parse_crypto_opt_chain(options, optarg);
1075
1076	/* Cipher options */
1077	else if (strcmp(lgopts[option_index].name, "cipher_algo") == 0)
1078		return parse_cipher_algo(&options->cipher_xform.cipher.algo,
1079				optarg);
1080
1081	else if (strcmp(lgopts[option_index].name, "cipher_op") == 0)
1082		return parse_cipher_op(&options->cipher_xform.cipher.op,
1083				optarg);
1084
1085	else if (strcmp(lgopts[option_index].name, "cipher_key") == 0) {
1086		options->ckey_param = 1;
1087		options->cipher_xform.cipher.key.length =
1088			parse_key(options->cipher_xform.cipher.key.data, optarg);
1089		if (options->cipher_xform.cipher.key.length > 0)
1090			return 0;
1091		else
1092			return -1;
1093	}
1094
1095	else if (strcmp(lgopts[option_index].name, "cipher_key_random_size") == 0)
1096		return parse_size(&options->ckey_random_size, optarg);
1097
1098	else if (strcmp(lgopts[option_index].name, "iv") == 0) {
1099		options->iv_param = 1;
1100		options->iv.length =
1101			parse_key(options->iv.data, optarg);
1102		if (options->iv.length > 0)
1103			return 0;
1104		else
1105			return -1;
1106	}
1107
1108	else if (strcmp(lgopts[option_index].name, "iv_random_size") == 0)
1109		return parse_size(&options->iv_random_size, optarg);
1110
1111	/* Authentication options */
1112	else if (strcmp(lgopts[option_index].name, "auth_algo") == 0) {
1113		return parse_auth_algo(&options->auth_xform.auth.algo,
1114				optarg);
1115	}
1116
1117	else if (strcmp(lgopts[option_index].name, "auth_op") == 0)
1118		return parse_auth_op(&options->auth_xform.auth.op,
1119				optarg);
1120
1121	else if (strcmp(lgopts[option_index].name, "auth_key") == 0) {
1122		options->akey_param = 1;
1123		options->auth_xform.auth.key.length =
1124			parse_key(options->auth_xform.auth.key.data, optarg);
1125		if (options->auth_xform.auth.key.length > 0)
1126			return 0;
1127		else
1128			return -1;
1129	}
1130
1131	else if (strcmp(lgopts[option_index].name, "auth_key_random_size") == 0) {
1132		return parse_size(&options->akey_random_size, optarg);
1133	}
1134
1135	else if (strcmp(lgopts[option_index].name, "aad") == 0) {
1136		options->aad_param = 1;
1137		options->aad.length =
1138			parse_key(options->aad.data, optarg);
1139		if (options->aad.length > 0)
1140			return 0;
1141		else
1142			return -1;
1143	}
1144
1145	else if (strcmp(lgopts[option_index].name, "aad_random_size") == 0) {
1146		return parse_size(&options->aad_random_size, optarg);
1147	}
1148
1149	else if (strcmp(lgopts[option_index].name, "digest_size") == 0) {
1150		return parse_size(&options->digest_size, optarg);
1151	}
1152
1153	else if (strcmp(lgopts[option_index].name, "sessionless") == 0) {
1154		options->sessionless = 1;
1155		return 0;
1156	}
1157
1158	return -1;
1159}
1160
1161/** Parse port mask */
1162static int
1163l2fwd_crypto_parse_portmask(struct l2fwd_crypto_options *options,
1164		const char *q_arg)
1165{
1166	char *end = NULL;
1167	unsigned long pm;
1168
1169	/* parse hexadecimal string */
1170	pm = strtoul(q_arg, &end, 16);
1171	if ((pm == '\0') || (end == NULL) || (*end != '\0'))
1172		pm = 0;
1173
1174	options->portmask = pm;
1175	if (options->portmask == 0) {
1176		printf("invalid portmask specified\n");
1177		return -1;
1178	}
1179
1180	return pm;
1181}
1182
1183/** Parse number of queues */
1184static int
1185l2fwd_crypto_parse_nqueue(struct l2fwd_crypto_options *options,
1186		const char *q_arg)
1187{
1188	char *end = NULL;
1189	unsigned long n;
1190
1191	/* parse hexadecimal string */
1192	n = strtoul(q_arg, &end, 10);
1193	if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0'))
1194		n = 0;
1195	else if (n >= MAX_RX_QUEUE_PER_LCORE)
1196		n = 0;
1197
1198	options->nb_ports_per_lcore = n;
1199	if (options->nb_ports_per_lcore == 0) {
1200		printf("invalid number of ports selected\n");
1201		return -1;
1202	}
1203
1204	return 0;
1205}
1206
1207/** Parse timer period */
1208static int
1209l2fwd_crypto_parse_timer_period(struct l2fwd_crypto_options *options,
1210		const char *q_arg)
1211{
1212	char *end = NULL;
1213	unsigned long n;
1214
1215	/* parse number string */
1216	n = (unsigned)strtol(q_arg, &end, 10);
1217	if ((q_arg[0] == '\0') || (end == NULL) || (*end != '\0'))
1218		n = 0;
1219
1220	if (n >= MAX_TIMER_PERIOD) {
1221		printf("Warning refresh period specified %lu is greater than "
1222				"max value %lu! using max value",
1223				n, MAX_TIMER_PERIOD);
1224		n = MAX_TIMER_PERIOD;
1225	}
1226
1227	options->refresh_period = n * 1000 * TIMER_MILLISECOND;
1228
1229	return 0;
1230}
1231
1232/** Generate default options for application */
1233static void
1234l2fwd_crypto_default_options(struct l2fwd_crypto_options *options)
1235{
1236	options->portmask = 0xffffffff;
1237	options->nb_ports_per_lcore = 1;
1238	options->refresh_period = 10000;
1239	options->single_lcore = 0;
1240	options->sessionless = 0;
1241
1242	options->xform_chain = L2FWD_CRYPTO_CIPHER_HASH;
1243
1244	/* Cipher Data */
1245	options->cipher_xform.type = RTE_CRYPTO_SYM_XFORM_CIPHER;
1246	options->cipher_xform.next = NULL;
1247	options->ckey_param = 0;
1248	options->ckey_random_size = -1;
1249	options->cipher_xform.cipher.key.length = 0;
1250	options->iv_param = 0;
1251	options->iv_random_size = -1;
1252	options->iv.length = 0;
1253
1254	options->cipher_xform.cipher.algo = RTE_CRYPTO_CIPHER_AES_CBC;
1255	options->cipher_xform.cipher.op = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
1256
1257	/* Authentication Data */
1258	options->auth_xform.type = RTE_CRYPTO_SYM_XFORM_AUTH;
1259	options->auth_xform.next = NULL;
1260	options->akey_param = 0;
1261	options->akey_random_size = -1;
1262	options->auth_xform.auth.key.length = 0;
1263	options->aad_param = 0;
1264	options->aad_random_size = -1;
1265	options->aad.length = 0;
1266	options->digest_size = -1;
1267
1268	options->auth_xform.auth.algo = RTE_CRYPTO_AUTH_SHA1_HMAC;
1269	options->auth_xform.auth.op = RTE_CRYPTO_AUTH_OP_GENERATE;
1270
1271	options->type = CDEV_TYPE_ANY;
1272}
1273
1274static void
1275display_cipher_info(struct l2fwd_crypto_options *options)
1276{
1277	printf("\n---- Cipher information ---\n");
1278	printf("Algorithm: %s\n",
1279		supported_cipher_algo[options->cipher_xform.cipher.algo]);
1280	rte_hexdump(stdout, "Cipher key:",
1281			options->cipher_xform.cipher.key.data,
1282			options->cipher_xform.cipher.key.length);
1283	rte_hexdump(stdout, "IV:", options->iv.data, options->iv.length);
1284}
1285
1286static void
1287display_auth_info(struct l2fwd_crypto_options *options)
1288{
1289	printf("\n---- Authentication information ---\n");
1290	printf("Algorithm: %s\n",
1291		supported_auth_algo[options->auth_xform.auth.algo]);
1292	rte_hexdump(stdout, "Auth key:",
1293			options->auth_xform.auth.key.data,
1294			options->auth_xform.auth.key.length);
1295	rte_hexdump(stdout, "AAD:", options->aad.data, options->aad.length);
1296}
1297
1298static void
1299l2fwd_crypto_options_print(struct l2fwd_crypto_options *options)
1300{
1301	char string_cipher_op[MAX_STR_LEN];
1302	char string_auth_op[MAX_STR_LEN];
1303
1304	if (options->cipher_xform.cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
1305		strcpy(string_cipher_op, "Encrypt");
1306	else
1307		strcpy(string_cipher_op, "Decrypt");
1308
1309	if (options->auth_xform.auth.op == RTE_CRYPTO_AUTH_OP_GENERATE)
1310		strcpy(string_auth_op, "Auth generate");
1311	else
1312		strcpy(string_auth_op, "Auth verify");
1313
1314	printf("Options:-\nn");
1315	printf("portmask: %x\n", options->portmask);
1316	printf("ports per lcore: %u\n", options->nb_ports_per_lcore);
1317	printf("refresh period : %u\n", options->refresh_period);
1318	printf("single lcore mode: %s\n",
1319			options->single_lcore ? "enabled" : "disabled");
1320	printf("stats_printing: %s\n",
1321			options->refresh_period == 0 ? "disabled" : "enabled");
1322
1323	printf("sessionless crypto: %s\n",
1324			options->sessionless ? "enabled" : "disabled");
1325
1326	if (options->ckey_param && (options->ckey_random_size != -1))
1327		printf("Cipher key already parsed, ignoring size of random key\n");
1328
1329	if (options->akey_param && (options->akey_random_size != -1))
1330		printf("Auth key already parsed, ignoring size of random key\n");
1331
1332	if (options->iv_param && (options->iv_random_size != -1))
1333		printf("IV already parsed, ignoring size of random IV\n");
1334
1335	if (options->aad_param && (options->aad_random_size != -1))
1336		printf("AAD already parsed, ignoring size of random AAD\n");
1337
1338	printf("\nCrypto chain: ");
1339	switch (options->xform_chain) {
1340	case L2FWD_CRYPTO_CIPHER_HASH:
1341		printf("Input --> %s --> %s --> Output\n",
1342			string_cipher_op, string_auth_op);
1343		display_cipher_info(options);
1344		display_auth_info(options);
1345		break;
1346	case L2FWD_CRYPTO_HASH_CIPHER:
1347		printf("Input --> %s --> %s --> Output\n",
1348			string_auth_op, string_cipher_op);
1349		display_cipher_info(options);
1350		display_auth_info(options);
1351		break;
1352	case L2FWD_CRYPTO_HASH_ONLY:
1353		printf("Input --> %s --> Output\n", string_auth_op);
1354		display_auth_info(options);
1355		break;
1356	case L2FWD_CRYPTO_CIPHER_ONLY:
1357		printf("Input --> %s --> Output\n", string_cipher_op);
1358		display_cipher_info(options);
1359		break;
1360	}
1361}
1362
1363/* Parse the argument given in the command line of the application */
1364static int
1365l2fwd_crypto_parse_args(struct l2fwd_crypto_options *options,
1366		int argc, char **argv)
1367{
1368	int opt, retval, option_index;
1369	char **argvopt = argv, *prgname = argv[0];
1370
1371	static struct option lgopts[] = {
1372			{ "sessionless", no_argument, 0, 0 },
1373
1374			{ "cdev_type", required_argument, 0, 0 },
1375			{ "chain", required_argument, 0, 0 },
1376
1377			{ "cipher_algo", required_argument, 0, 0 },
1378			{ "cipher_op", required_argument, 0, 0 },
1379			{ "cipher_key", required_argument, 0, 0 },
1380			{ "cipher_key_random_size", required_argument, 0, 0 },
1381
1382			{ "auth_algo", required_argument, 0, 0 },
1383			{ "auth_op", required_argument, 0, 0 },
1384			{ "auth_key", required_argument, 0, 0 },
1385			{ "auth_key_random_size", required_argument, 0, 0 },
1386
1387			{ "iv", required_argument, 0, 0 },
1388			{ "iv_random_size", required_argument, 0, 0 },
1389			{ "aad", required_argument, 0, 0 },
1390			{ "aad_random_size", required_argument, 0, 0 },
1391			{ "digest_size", required_argument, 0, 0 },
1392
1393			{ "sessionless", no_argument, 0, 0 },
1394
1395			{ NULL, 0, 0, 0 }
1396	};
1397
1398	l2fwd_crypto_default_options(options);
1399
1400	while ((opt = getopt_long(argc, argvopt, "p:q:sT:", lgopts,
1401			&option_index)) != EOF) {
1402		switch (opt) {
1403		/* long options */
1404		case 0:
1405			retval = l2fwd_crypto_parse_args_long_options(options,
1406					lgopts, option_index);
1407			if (retval < 0) {
1408				l2fwd_crypto_usage(prgname);
1409				return -1;
1410			}
1411			break;
1412
1413		/* portmask */
1414		case 'p':
1415			retval = l2fwd_crypto_parse_portmask(options, optarg);
1416			if (retval < 0) {
1417				l2fwd_crypto_usage(prgname);
1418				return -1;
1419			}
1420			break;
1421
1422		/* nqueue */
1423		case 'q':
1424			retval = l2fwd_crypto_parse_nqueue(options, optarg);
1425			if (retval < 0) {
1426				l2fwd_crypto_usage(prgname);
1427				return -1;
1428			}
1429			break;
1430
1431		/* single  */
1432		case 's':
1433			options->single_lcore = 1;
1434
1435			break;
1436
1437		/* timer period */
1438		case 'T':
1439			retval = l2fwd_crypto_parse_timer_period(options,
1440					optarg);
1441			if (retval < 0) {
1442				l2fwd_crypto_usage(prgname);
1443				return -1;
1444			}
1445			break;
1446
1447		default:
1448			l2fwd_crypto_usage(prgname);
1449			return -1;
1450		}
1451	}
1452
1453
1454	if (optind >= 0)
1455		argv[optind-1] = prgname;
1456
1457	retval = optind-1;
1458	optind = 0; /* reset getopt lib */
1459
1460	return retval;
1461}
1462
1463/* Check the link status of all ports in up to 9s, and print them finally */
1464static void
1465check_all_ports_link_status(uint8_t port_num, uint32_t port_mask)
1466{
1467#define CHECK_INTERVAL 100 /* 100ms */
1468#define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
1469	uint8_t portid, count, all_ports_up, print_flag = 0;
1470	struct rte_eth_link link;
1471
1472	printf("\nChecking link status");
1473	fflush(stdout);
1474	for (count = 0; count <= MAX_CHECK_TIME; count++) {
1475		all_ports_up = 1;
1476		for (portid = 0; portid < port_num; portid++) {
1477			if ((port_mask & (1 << portid)) == 0)
1478				continue;
1479			memset(&link, 0, sizeof(link));
1480			rte_eth_link_get_nowait(portid, &link);
1481			/* print link status if flag set */
1482			if (print_flag == 1) {
1483				if (link.link_status)
1484					printf("Port %d Link Up - speed %u "
1485						"Mbps - %s\n", (uint8_t)portid,
1486						(unsigned)link.link_speed,
1487				(link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
1488					("full-duplex") : ("half-duplex\n"));
1489				else
1490					printf("Port %d Link Down\n",
1491						(uint8_t)portid);
1492				continue;
1493			}
1494			/* clear all_ports_up flag if any link down */
1495			if (link.link_status == ETH_LINK_DOWN) {
1496				all_ports_up = 0;
1497				break;
1498			}
1499		}
1500		/* after finally printing all link status, get out */
1501		if (print_flag == 1)
1502			break;
1503
1504		if (all_ports_up == 0) {
1505			printf(".");
1506			fflush(stdout);
1507			rte_delay_ms(CHECK_INTERVAL);
1508		}
1509
1510		/* set the print_flag if all ports up or timeout */
1511		if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) {
1512			print_flag = 1;
1513			printf("done\n");
1514		}
1515	}
1516}
1517
1518/* Check if device has to be HW/SW or any */
1519static int
1520check_type(struct l2fwd_crypto_options *options, struct rte_cryptodev_info *dev_info)
1521{
1522	if (options->type == CDEV_TYPE_HW &&
1523			(dev_info->feature_flags & RTE_CRYPTODEV_FF_HW_ACCELERATED))
1524		return 0;
1525	if (options->type == CDEV_TYPE_SW &&
1526			!(dev_info->feature_flags & RTE_CRYPTODEV_FF_HW_ACCELERATED))
1527		return 0;
1528	if (options->type == CDEV_TYPE_ANY)
1529		return 0;
1530
1531	return -1;
1532}
1533
1534static inline int
1535check_supported_size(uint16_t length, uint16_t min, uint16_t max,
1536		uint16_t increment)
1537{
1538	uint16_t supp_size;
1539
1540	/* Single value */
1541	if (increment == 0) {
1542		if (length == min)
1543			return 0;
1544		else
1545			return -1;
1546	}
1547
1548	/* Range of values */
1549	for (supp_size = min; supp_size <= max; supp_size += increment) {
1550		if (length == supp_size)
1551			return 0;
1552	}
1553
1554	return -1;
1555}
1556static int
1557initialize_cryptodevs(struct l2fwd_crypto_options *options, unsigned nb_ports,
1558		uint8_t *enabled_cdevs)
1559{
1560	unsigned i, cdev_id, cdev_count, enabled_cdev_count = 0;
1561	const struct rte_cryptodev_capabilities *cap;
1562	enum rte_crypto_auth_algorithm cap_auth_algo;
1563	enum rte_crypto_auth_algorithm opt_auth_algo;
1564	enum rte_crypto_cipher_algorithm cap_cipher_algo;
1565	enum rte_crypto_cipher_algorithm opt_cipher_algo;
1566	int retval;
1567
1568	cdev_count = rte_cryptodev_count();
1569	if (cdev_count == 0) {
1570		printf("No crypto devices available\n");
1571		return -1;
1572	}
1573
1574	for (cdev_id = 0; cdev_id < cdev_count && enabled_cdev_count < nb_ports;
1575			cdev_id++) {
1576		struct rte_cryptodev_qp_conf qp_conf;
1577		struct rte_cryptodev_info dev_info;
1578
1579		struct rte_cryptodev_config conf = {
1580			.nb_queue_pairs = 1,
1581			.socket_id = SOCKET_ID_ANY,
1582			.session_mp = {
1583				.nb_objs = 2048,
1584				.cache_size = 64
1585			}
1586		};
1587
1588		rte_cryptodev_info_get(cdev_id, &dev_info);
1589
1590		/* Set cipher parameters */
1591		if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH ||
1592				options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER ||
1593				options->xform_chain == L2FWD_CRYPTO_CIPHER_ONLY) {
1594			/* Check if device supports cipher algo */
1595			i = 0;
1596			opt_cipher_algo = options->cipher_xform.cipher.algo;
1597			cap = &dev_info.capabilities[i];
1598			while (cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED) {
1599				cap_cipher_algo = cap->sym.cipher.algo;
1600				if (cap->sym.xform_type ==
1601						RTE_CRYPTO_SYM_XFORM_CIPHER) {
1602					if (cap_cipher_algo == opt_cipher_algo) {
1603						if (check_type(options, &dev_info) == 0)
1604							break;
1605					}
1606				}
1607				cap = &dev_info.capabilities[++i];
1608			}
1609
1610			if (cap->op == RTE_CRYPTO_OP_TYPE_UNDEFINED) {
1611				printf("Algorithm %s not supported by cryptodev %u"
1612					" or device not of preferred type (%s)\n",
1613					supported_cipher_algo[opt_cipher_algo],
1614					cdev_id,
1615					options->string_type);
1616				continue;
1617			}
1618
1619			options->block_size = cap->sym.cipher.block_size;
1620			/*
1621			 * Check if length of provided IV is supported
1622			 * by the algorithm chosen.
1623			 */
1624			if (options->iv_param) {
1625				if (check_supported_size(options->iv.length,
1626						cap->sym.cipher.iv_size.min,
1627						cap->sym.cipher.iv_size.max,
1628						cap->sym.cipher.iv_size.increment)
1629							!= 0) {
1630					printf("Unsupported IV length\n");
1631					return -1;
1632				}
1633			/*
1634			 * Check if length of IV to be randomly generated
1635			 * is supported by the algorithm chosen.
1636			 */
1637			} else if (options->iv_random_size != -1) {
1638				if (check_supported_size(options->iv_random_size,
1639						cap->sym.cipher.iv_size.min,
1640						cap->sym.cipher.iv_size.max,
1641						cap->sym.cipher.iv_size.increment)
1642							!= 0) {
1643					printf("Unsupported IV length\n");
1644					return -1;
1645				}
1646				options->iv.length = options->iv_random_size;
1647			/* No size provided, use minimum size. */
1648			} else
1649				options->iv.length = cap->sym.cipher.iv_size.min;
1650
1651			/*
1652			 * Check if length of provided cipher key is supported
1653			 * by the algorithm chosen.
1654			 */
1655			if (options->ckey_param) {
1656				if (check_supported_size(
1657						options->cipher_xform.cipher.key.length,
1658						cap->sym.cipher.key_size.min,
1659						cap->sym.cipher.key_size.max,
1660						cap->sym.cipher.key_size.increment)
1661							!= 0) {
1662					printf("Unsupported cipher key length\n");
1663					return -1;
1664				}
1665			/*
1666			 * Check if length of the cipher key to be randomly generated
1667			 * is supported by the algorithm chosen.
1668			 */
1669			} else if (options->ckey_random_size != -1) {
1670				if (check_supported_size(options->ckey_random_size,
1671						cap->sym.cipher.key_size.min,
1672						cap->sym.cipher.key_size.max,
1673						cap->sym.cipher.key_size.increment)
1674							!= 0) {
1675					printf("Unsupported cipher key length\n");
1676					return -1;
1677				}
1678				options->cipher_xform.cipher.key.length =
1679							options->ckey_random_size;
1680			/* No size provided, use minimum size. */
1681			} else
1682				options->cipher_xform.cipher.key.length =
1683						cap->sym.cipher.key_size.min;
1684
1685			if (!options->ckey_param)
1686				generate_random_key(
1687					options->cipher_xform.cipher.key.data,
1688					options->cipher_xform.cipher.key.length);
1689
1690		}
1691
1692		/* Set auth parameters */
1693		if (options->xform_chain == L2FWD_CRYPTO_CIPHER_HASH ||
1694				options->xform_chain == L2FWD_CRYPTO_HASH_CIPHER ||
1695				options->xform_chain == L2FWD_CRYPTO_HASH_ONLY) {
1696			/* Check if device supports auth algo */
1697			i = 0;
1698			opt_auth_algo = options->auth_xform.auth.algo;
1699			cap = &dev_info.capabilities[i];
1700			while (cap->op != RTE_CRYPTO_OP_TYPE_UNDEFINED) {
1701				cap_auth_algo = cap->sym.auth.algo;
1702				if ((cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AUTH) &&
1703						(cap_auth_algo == opt_auth_algo) &&
1704						(check_type(options, &dev_info) == 0)) {
1705					break;
1706				}
1707				cap = &dev_info.capabilities[++i];
1708			}
1709
1710			if (cap->op == RTE_CRYPTO_OP_TYPE_UNDEFINED) {
1711				printf("Algorithm %s not supported by cryptodev %u"
1712					" or device not of preferred type (%s)\n",
1713					supported_auth_algo[opt_auth_algo],
1714					cdev_id,
1715					options->string_type);
1716				continue;
1717			}
1718
1719			/*
1720			 * Check if length of provided AAD is supported
1721			 * by the algorithm chosen.
1722			 */
1723			if (options->aad_param) {
1724				if (check_supported_size(options->aad.length,
1725						cap->sym.auth.aad_size.min,
1726						cap->sym.auth.aad_size.max,
1727						cap->sym.auth.aad_size.increment)
1728							!= 0) {
1729					printf("Unsupported AAD length\n");
1730					return -1;
1731				}
1732			/*
1733			 * Check if length of AAD to be randomly generated
1734			 * is supported by the algorithm chosen.
1735			 */
1736			} else if (options->aad_random_size != -1) {
1737				if (check_supported_size(options->aad_random_size,
1738						cap->sym.auth.aad_size.min,
1739						cap->sym.auth.aad_size.max,
1740						cap->sym.auth.aad_size.increment)
1741							!= 0) {
1742					printf("Unsupported AAD length\n");
1743					return -1;
1744				}
1745				options->aad.length = options->aad_random_size;
1746			/* No size provided, use minimum size. */
1747			} else
1748				options->aad.length = cap->sym.auth.aad_size.min;
1749
1750			options->auth_xform.auth.add_auth_data_length =
1751						options->aad.length;
1752
1753			/*
1754			 * Check if length of provided auth key is supported
1755			 * by the algorithm chosen.
1756			 */
1757			if (options->akey_param) {
1758				if (check_supported_size(
1759						options->auth_xform.auth.key.length,
1760						cap->sym.auth.key_size.min,
1761						cap->sym.auth.key_size.max,
1762						cap->sym.auth.key_size.increment)
1763							!= 0) {
1764					printf("Unsupported auth key length\n");
1765					return -1;
1766				}
1767			/*
1768			 * Check if length of the auth key to be randomly generated
1769			 * is supported by the algorithm chosen.
1770			 */
1771			} else if (options->akey_random_size != -1) {
1772				if (check_supported_size(options->akey_random_size,
1773						cap->sym.auth.key_size.min,
1774						cap->sym.auth.key_size.max,
1775						cap->sym.auth.key_size.increment)
1776							!= 0) {
1777					printf("Unsupported auth key length\n");
1778					return -1;
1779				}
1780				options->auth_xform.auth.key.length =
1781							options->akey_random_size;
1782			/* No size provided, use minimum size. */
1783			} else
1784				options->auth_xform.auth.key.length =
1785						cap->sym.auth.key_size.min;
1786
1787			if (!options->akey_param)
1788				generate_random_key(
1789					options->auth_xform.auth.key.data,
1790					options->auth_xform.auth.key.length);
1791
1792			/* Check if digest size is supported by the algorithm. */
1793			if (options->digest_size != -1) {
1794				if (check_supported_size(options->digest_size,
1795						cap->sym.auth.digest_size.min,
1796						cap->sym.auth.digest_size.max,
1797						cap->sym.auth.digest_size.increment)
1798							!= 0) {
1799					printf("Unsupported digest length\n");
1800					return -1;
1801				}
1802				options->auth_xform.auth.digest_length =
1803							options->digest_size;
1804			/* No size provided, use minimum size. */
1805			} else
1806				options->auth_xform.auth.digest_length =
1807						cap->sym.auth.digest_size.min;
1808		}
1809
1810		retval = rte_cryptodev_configure(cdev_id, &conf);
1811		if (retval < 0) {
1812			printf("Failed to configure cryptodev %u", cdev_id);
1813			return -1;
1814		}
1815
1816		qp_conf.nb_descriptors = 2048;
1817
1818		retval = rte_cryptodev_queue_pair_setup(cdev_id, 0, &qp_conf,
1819				SOCKET_ID_ANY);
1820		if (retval < 0) {
1821			printf("Failed to setup queue pair %u on cryptodev %u",
1822					0, cdev_id);
1823			return -1;
1824		}
1825
1826		retval = rte_cryptodev_start(cdev_id);
1827		if (retval < 0) {
1828			printf("Failed to start device %u: error %d\n",
1829					cdev_id, retval);
1830			return -1;
1831		}
1832
1833		l2fwd_enabled_crypto_mask |= (((uint64_t)1) << cdev_id);
1834
1835		enabled_cdevs[cdev_id] = 1;
1836		enabled_cdev_count++;
1837	}
1838
1839	return enabled_cdev_count;
1840}
1841
1842static int
1843initialize_ports(struct l2fwd_crypto_options *options)
1844{
1845	uint8_t last_portid, portid;
1846	unsigned enabled_portcount = 0;
1847	unsigned nb_ports = rte_eth_dev_count();
1848
1849	if (nb_ports == 0) {
1850		printf("No Ethernet ports - bye\n");
1851		return -1;
1852	}
1853
1854	/* Reset l2fwd_dst_ports */
1855	for (portid = 0; portid < RTE_MAX_ETHPORTS; portid++)
1856		l2fwd_dst_ports[portid] = 0;
1857
1858	for (last_portid = 0, portid = 0; portid < nb_ports; portid++) {
1859		int retval;
1860
1861		/* Skip ports that are not enabled */
1862		if ((options->portmask & (1 << portid)) == 0)
1863			continue;
1864
1865		/* init port */
1866		printf("Initializing port %u... ", (unsigned) portid);
1867		fflush(stdout);
1868		retval = rte_eth_dev_configure(portid, 1, 1, &port_conf);
1869		if (retval < 0) {
1870			printf("Cannot configure device: err=%d, port=%u\n",
1871				  retval, (unsigned) portid);
1872			return -1;
1873		}
1874
1875		/* init one RX queue */
1876		fflush(stdout);
1877		retval = rte_eth_rx_queue_setup(portid, 0, nb_rxd,
1878					     rte_eth_dev_socket_id(portid),
1879					     NULL, l2fwd_pktmbuf_pool);
1880		if (retval < 0) {
1881			printf("rte_eth_rx_queue_setup:err=%d, port=%u\n",
1882					retval, (unsigned) portid);
1883			return -1;
1884		}
1885
1886		/* init one TX queue on each port */
1887		fflush(stdout);
1888		retval = rte_eth_tx_queue_setup(portid, 0, nb_txd,
1889				rte_eth_dev_socket_id(portid),
1890				NULL);
1891		if (retval < 0) {
1892			printf("rte_eth_tx_queue_setup:err=%d, port=%u\n",
1893				retval, (unsigned) portid);
1894
1895			return -1;
1896		}
1897
1898		/* Start device */
1899		retval = rte_eth_dev_start(portid);
1900		if (retval < 0) {
1901			printf("rte_eth_dev_start:err=%d, port=%u\n",
1902					retval, (unsigned) portid);
1903			return -1;
1904		}
1905
1906		rte_eth_promiscuous_enable(portid);
1907
1908		rte_eth_macaddr_get(portid, &l2fwd_ports_eth_addr[portid]);
1909
1910		printf("Port %u, MAC address: %02X:%02X:%02X:%02X:%02X:%02X\n\n",
1911				(unsigned) portid,
1912				l2fwd_ports_eth_addr[portid].addr_bytes[0],
1913				l2fwd_ports_eth_addr[portid].addr_bytes[1],
1914				l2fwd_ports_eth_addr[portid].addr_bytes[2],
1915				l2fwd_ports_eth_addr[portid].addr_bytes[3],
1916				l2fwd_ports_eth_addr[portid].addr_bytes[4],
1917				l2fwd_ports_eth_addr[portid].addr_bytes[5]);
1918
1919		/* initialize port stats */
1920		memset(&port_statistics, 0, sizeof(port_statistics));
1921
1922		/* Setup port forwarding table */
1923		if (enabled_portcount % 2) {
1924			l2fwd_dst_ports[portid] = last_portid;
1925			l2fwd_dst_ports[last_portid] = portid;
1926		} else {
1927			last_portid = portid;
1928		}
1929
1930		l2fwd_enabled_port_mask |= (1 << portid);
1931		enabled_portcount++;
1932	}
1933
1934	if (enabled_portcount == 1) {
1935		l2fwd_dst_ports[last_portid] = last_portid;
1936	} else if (enabled_portcount % 2) {
1937		printf("odd number of ports in portmask- bye\n");
1938		return -1;
1939	}
1940
1941	check_all_ports_link_status(nb_ports, l2fwd_enabled_port_mask);
1942
1943	return enabled_portcount;
1944}
1945
1946static void
1947reserve_key_memory(struct l2fwd_crypto_options *options)
1948{
1949	options->cipher_xform.cipher.key.data = rte_malloc("crypto key",
1950						MAX_KEY_SIZE, 0);
1951	if (options->cipher_xform.cipher.key.data == NULL)
1952		rte_exit(EXIT_FAILURE, "Failed to allocate memory for cipher key");
1953
1954
1955	options->auth_xform.auth.key.data = rte_malloc("auth key",
1956						MAX_KEY_SIZE, 0);
1957	if (options->auth_xform.auth.key.data == NULL)
1958		rte_exit(EXIT_FAILURE, "Failed to allocate memory for auth key");
1959
1960	options->iv.data = rte_malloc("iv", MAX_KEY_SIZE, 0);
1961	if (options->iv.data == NULL)
1962		rte_exit(EXIT_FAILURE, "Failed to allocate memory for IV");
1963	options->iv.phys_addr = rte_malloc_virt2phy(options->iv.data);
1964
1965	options->aad.data = rte_malloc("aad", MAX_KEY_SIZE, 0);
1966	if (options->aad.data == NULL)
1967		rte_exit(EXIT_FAILURE, "Failed to allocate memory for AAD");
1968	options->aad.phys_addr = rte_malloc_virt2phy(options->aad.data);
1969}
1970
1971int
1972main(int argc, char **argv)
1973{
1974	struct lcore_queue_conf *qconf;
1975	struct l2fwd_crypto_options options;
1976
1977	uint8_t nb_ports, nb_cryptodevs, portid, cdev_id;
1978	unsigned lcore_id, rx_lcore_id;
1979	int ret, enabled_cdevcount, enabled_portcount;
1980	uint8_t enabled_cdevs[RTE_CRYPTO_MAX_DEVS] = {0};
1981
1982	/* init EAL */
1983	ret = rte_eal_init(argc, argv);
1984	if (ret < 0)
1985		rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n");
1986	argc -= ret;
1987	argv += ret;
1988
1989	/* reserve memory for Cipher/Auth key and IV */
1990	reserve_key_memory(&options);
1991
1992	/* fill out the supported algorithm tables */
1993	fill_supported_algorithm_tables();
1994
1995	/* parse application arguments (after the EAL ones) */
1996	ret = l2fwd_crypto_parse_args(&options, argc, argv);
1997	if (ret < 0)
1998		rte_exit(EXIT_FAILURE, "Invalid L2FWD-CRYPTO arguments\n");
1999
2000	/* create the mbuf pool */
2001	l2fwd_pktmbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", NB_MBUF, 512,
2002			sizeof(struct rte_crypto_op),
2003			RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());
2004	if (l2fwd_pktmbuf_pool == NULL)
2005		rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n");
2006
2007	/* create crypto op pool */
2008	l2fwd_crypto_op_pool = rte_crypto_op_pool_create("crypto_op_pool",
2009			RTE_CRYPTO_OP_TYPE_SYMMETRIC, NB_MBUF, 128, 0,
2010			rte_socket_id());
2011	if (l2fwd_crypto_op_pool == NULL)
2012		rte_exit(EXIT_FAILURE, "Cannot create crypto op pool\n");
2013
2014	/* Enable Ethernet ports */
2015	enabled_portcount = initialize_ports(&options);
2016	if (enabled_portcount < 1)
2017		rte_exit(EXIT_FAILURE, "Failed to initial Ethernet ports\n");
2018
2019	nb_ports = rte_eth_dev_count();
2020	/* Initialize the port/queue configuration of each logical core */
2021	for (rx_lcore_id = 0, qconf = NULL, portid = 0;
2022			portid < nb_ports; portid++) {
2023
2024		/* skip ports that are not enabled */
2025		if ((options.portmask & (1 << portid)) == 0)
2026			continue;
2027
2028		if (options.single_lcore && qconf == NULL) {
2029			while (rte_lcore_is_enabled(rx_lcore_id) == 0) {
2030				rx_lcore_id++;
2031				if (rx_lcore_id >= RTE_MAX_LCORE)
2032					rte_exit(EXIT_FAILURE,
2033							"Not enough cores\n");
2034			}
2035		} else if (!options.single_lcore) {
2036			/* get the lcore_id for this port */
2037			while (rte_lcore_is_enabled(rx_lcore_id) == 0 ||
2038			       lcore_queue_conf[rx_lcore_id].nb_rx_ports ==
2039			       options.nb_ports_per_lcore) {
2040				rx_lcore_id++;
2041				if (rx_lcore_id >= RTE_MAX_LCORE)
2042					rte_exit(EXIT_FAILURE,
2043							"Not enough cores\n");
2044			}
2045		}
2046
2047		/* Assigned a new logical core in the loop above. */
2048		if (qconf != &lcore_queue_conf[rx_lcore_id])
2049			qconf = &lcore_queue_conf[rx_lcore_id];
2050
2051		qconf->rx_port_list[qconf->nb_rx_ports] = portid;
2052		qconf->nb_rx_ports++;
2053
2054		printf("Lcore %u: RX port %u\n", rx_lcore_id, (unsigned)portid);
2055	}
2056
2057	/* Enable Crypto devices */
2058	enabled_cdevcount = initialize_cryptodevs(&options, enabled_portcount,
2059			enabled_cdevs);
2060	if (enabled_cdevcount < 0)
2061		rte_exit(EXIT_FAILURE, "Failed to initialize crypto devices\n");
2062
2063	if (enabled_cdevcount < enabled_portcount)
2064		rte_exit(EXIT_FAILURE, "Number of capable crypto devices (%d) "
2065				"has to be more or equal to number of ports (%d)\n",
2066				enabled_cdevcount, enabled_portcount);
2067
2068	nb_cryptodevs = rte_cryptodev_count();
2069
2070	/* Initialize the port/cryptodev configuration of each logical core */
2071	for (rx_lcore_id = 0, qconf = NULL, cdev_id = 0;
2072			cdev_id < nb_cryptodevs && enabled_cdevcount;
2073			cdev_id++) {
2074		/* Crypto op not supported by crypto device */
2075		if (!enabled_cdevs[cdev_id])
2076			continue;
2077
2078		if (options.single_lcore && qconf == NULL) {
2079			while (rte_lcore_is_enabled(rx_lcore_id) == 0) {
2080				rx_lcore_id++;
2081				if (rx_lcore_id >= RTE_MAX_LCORE)
2082					rte_exit(EXIT_FAILURE,
2083							"Not enough cores\n");
2084			}
2085		} else if (!options.single_lcore) {
2086			/* get the lcore_id for this port */
2087			while (rte_lcore_is_enabled(rx_lcore_id) == 0 ||
2088			       lcore_queue_conf[rx_lcore_id].nb_crypto_devs ==
2089			       options.nb_ports_per_lcore) {
2090				rx_lcore_id++;
2091				if (rx_lcore_id >= RTE_MAX_LCORE)
2092					rte_exit(EXIT_FAILURE,
2093							"Not enough cores\n");
2094			}
2095		}
2096
2097		/* Assigned a new logical core in the loop above. */
2098		if (qconf != &lcore_queue_conf[rx_lcore_id])
2099			qconf = &lcore_queue_conf[rx_lcore_id];
2100
2101		qconf->cryptodev_list[qconf->nb_crypto_devs] = cdev_id;
2102		qconf->nb_crypto_devs++;
2103
2104		enabled_cdevcount--;
2105
2106		printf("Lcore %u: cryptodev %u\n", rx_lcore_id,
2107				(unsigned)cdev_id);
2108	}
2109
2110	/* launch per-lcore init on every lcore */
2111	rte_eal_mp_remote_launch(l2fwd_launch_one_lcore, (void *)&options,
2112			CALL_MASTER);
2113	RTE_LCORE_FOREACH_SLAVE(lcore_id) {
2114		if (rte_eal_wait_lcore(lcore_id) < 0)
2115			return -1;
2116	}
2117
2118	return 0;
2119}
2120