rte_eth_szedata2.c revision 43192222
1/*-
2 *   BSD LICENSE
3 *
4 *   Copyright (c) 2015 - 2016 CESNET
5 *   All rights reserved.
6 *
7 *   Redistribution and use in source and binary forms, with or without
8 *   modification, are permitted provided that the following conditions
9 *   are met:
10 *
11 *     * Redistributions of source code must retain the above copyright
12 *       notice, this list of conditions and the following disclaimer.
13 *     * Redistributions in binary form must reproduce the above copyright
14 *       notice, this list of conditions and the following disclaimer in
15 *       the documentation and/or other materials provided with the
16 *       distribution.
17 *     * Neither the name of CESNET nor the names of its
18 *       contributors may be used to endorse or promote products derived
19 *       from this software without specific prior written permission.
20 *
21 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34#include <stdint.h>
35#include <unistd.h>
36#include <stdbool.h>
37#include <err.h>
38#include <sys/types.h>
39#include <dirent.h>
40#include <sys/stat.h>
41#include <fcntl.h>
42#include <sys/mman.h>
43
44#include <libsze2.h>
45
46#include <rte_mbuf.h>
47#include <rte_ethdev.h>
48#include <rte_malloc.h>
49#include <rte_memcpy.h>
50#include <rte_kvargs.h>
51#include <rte_dev.h>
52#include <rte_atomic.h>
53
54#include "rte_eth_szedata2.h"
55
56#define RTE_ETH_SZEDATA2_MAX_RX_QUEUES 32
57#define RTE_ETH_SZEDATA2_MAX_TX_QUEUES 32
58#define RTE_ETH_SZEDATA2_TX_LOCK_SIZE (32 * 1024 * 1024)
59
60/**
61 * size of szedata2_packet header with alignment
62 */
63#define RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED 8
64
65#define RTE_SZEDATA2_DRIVER_NAME net_szedata2
66#define RTE_SZEDATA2_PCI_DRIVER_NAME "rte_szedata2_pmd"
67
68#define SZEDATA2_DEV_PATH_FMT "/dev/szedataII%u"
69
70struct szedata2_rx_queue {
71	struct szedata *sze;
72	uint8_t rx_channel;
73	uint8_t in_port;
74	struct rte_mempool *mb_pool;
75	volatile uint64_t rx_pkts;
76	volatile uint64_t rx_bytes;
77	volatile uint64_t err_pkts;
78};
79
80struct szedata2_tx_queue {
81	struct szedata *sze;
82	uint8_t tx_channel;
83	volatile uint64_t tx_pkts;
84	volatile uint64_t tx_bytes;
85	volatile uint64_t err_pkts;
86};
87
88struct pmd_internals {
89	struct szedata2_rx_queue rx_queue[RTE_ETH_SZEDATA2_MAX_RX_QUEUES];
90	struct szedata2_tx_queue tx_queue[RTE_ETH_SZEDATA2_MAX_TX_QUEUES];
91	uint16_t max_rx_queues;
92	uint16_t max_tx_queues;
93	char sze_dev[PATH_MAX];
94};
95
96static struct ether_addr eth_addr = {
97	.addr_bytes = { 0x00, 0x11, 0x17, 0x00, 0x00, 0x00 }
98};
99
100static uint16_t
101eth_szedata2_rx(void *queue,
102		struct rte_mbuf **bufs,
103		uint16_t nb_pkts)
104{
105	unsigned int i;
106	struct rte_mbuf *mbuf;
107	struct szedata2_rx_queue *sze_q = queue;
108	struct rte_pktmbuf_pool_private *mbp_priv;
109	uint16_t num_rx = 0;
110	uint16_t buf_size;
111	uint16_t sg_size;
112	uint16_t hw_size;
113	uint16_t packet_size;
114	uint64_t num_bytes = 0;
115	struct szedata *sze = sze_q->sze;
116	uint8_t *header_ptr = NULL; /* header of packet */
117	uint8_t *packet_ptr1 = NULL;
118	uint8_t *packet_ptr2 = NULL;
119	uint16_t packet_len1 = 0;
120	uint16_t packet_len2 = 0;
121	uint16_t hw_data_align;
122
123	if (unlikely(sze_q->sze == NULL || nb_pkts == 0))
124		return 0;
125
126	/*
127	 * Reads the given number of packets from szedata2 channel given
128	 * by queue and copies the packet data into a newly allocated mbuf
129	 * to return.
130	 */
131	for (i = 0; i < nb_pkts; i++) {
132		mbuf = rte_pktmbuf_alloc(sze_q->mb_pool);
133
134		if (unlikely(mbuf == NULL))
135			break;
136
137		/* get the next sze packet */
138		if (sze->ct_rx_lck != NULL && !sze->ct_rx_rem_bytes &&
139				sze->ct_rx_lck->next == NULL) {
140			/* unlock old data */
141			szedata_rx_unlock_data(sze_q->sze, sze->ct_rx_lck_orig);
142			sze->ct_rx_lck_orig = NULL;
143			sze->ct_rx_lck = NULL;
144		}
145
146		if (!sze->ct_rx_rem_bytes && sze->ct_rx_lck_orig == NULL) {
147			/* nothing to read, lock new data */
148			sze->ct_rx_lck = szedata_rx_lock_data(sze_q->sze, ~0U);
149			sze->ct_rx_lck_orig = sze->ct_rx_lck;
150
151			if (sze->ct_rx_lck == NULL) {
152				/* nothing to lock */
153				rte_pktmbuf_free(mbuf);
154				break;
155			}
156
157			sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
158			sze->ct_rx_rem_bytes = sze->ct_rx_lck->len;
159
160			if (!sze->ct_rx_rem_bytes) {
161				rte_pktmbuf_free(mbuf);
162				break;
163			}
164		}
165
166		if (sze->ct_rx_rem_bytes < RTE_SZE2_PACKET_HEADER_SIZE) {
167			/*
168			 * cut in header
169			 * copy parts of header to merge buffer
170			 */
171			if (sze->ct_rx_lck->next == NULL) {
172				rte_pktmbuf_free(mbuf);
173				break;
174			}
175
176			/* copy first part of header */
177			rte_memcpy(sze->ct_rx_buffer, sze->ct_rx_cur_ptr,
178					sze->ct_rx_rem_bytes);
179
180			/* copy second part of header */
181			sze->ct_rx_lck = sze->ct_rx_lck->next;
182			sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
183			rte_memcpy(sze->ct_rx_buffer + sze->ct_rx_rem_bytes,
184				sze->ct_rx_cur_ptr,
185				RTE_SZE2_PACKET_HEADER_SIZE -
186				sze->ct_rx_rem_bytes);
187
188			sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE -
189				sze->ct_rx_rem_bytes;
190			sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
191				RTE_SZE2_PACKET_HEADER_SIZE +
192				sze->ct_rx_rem_bytes;
193
194			header_ptr = (uint8_t *)sze->ct_rx_buffer;
195		} else {
196			/* not cut */
197			header_ptr = (uint8_t *)sze->ct_rx_cur_ptr;
198			sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE;
199			sze->ct_rx_rem_bytes -= RTE_SZE2_PACKET_HEADER_SIZE;
200		}
201
202		sg_size = le16toh(*((uint16_t *)header_ptr));
203		hw_size = le16toh(*(((uint16_t *)header_ptr) + 1));
204		packet_size = sg_size -
205			RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size);
206
207
208		/* checks if packet all right */
209		if (!sg_size)
210			errx(5, "Zero segsize");
211
212		/* check sg_size and hwsize */
213		if (hw_size > sg_size - RTE_SZE2_PACKET_HEADER_SIZE) {
214			errx(10, "Hwsize bigger than expected. Segsize: %d, "
215				"hwsize: %d", sg_size, hw_size);
216		}
217
218		hw_data_align =
219			RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size) -
220			RTE_SZE2_PACKET_HEADER_SIZE;
221
222		if (sze->ct_rx_rem_bytes >=
223				(uint16_t)(sg_size -
224				RTE_SZE2_PACKET_HEADER_SIZE)) {
225			/* no cut */
226			/* one packet ready - go to another */
227			packet_ptr1 = sze->ct_rx_cur_ptr + hw_data_align;
228			packet_len1 = packet_size;
229			packet_ptr2 = NULL;
230			packet_len2 = 0;
231
232			sze->ct_rx_cur_ptr += RTE_SZE2_ALIGN8(sg_size) -
233				RTE_SZE2_PACKET_HEADER_SIZE;
234			sze->ct_rx_rem_bytes -= RTE_SZE2_ALIGN8(sg_size) -
235				RTE_SZE2_PACKET_HEADER_SIZE;
236		} else {
237			/* cut in data */
238			if (sze->ct_rx_lck->next == NULL) {
239				errx(6, "Need \"next\" lock, "
240					"but it is missing: %u",
241					sze->ct_rx_rem_bytes);
242			}
243
244			/* skip hw data */
245			if (sze->ct_rx_rem_bytes <= hw_data_align) {
246				uint16_t rem_size = hw_data_align -
247					sze->ct_rx_rem_bytes;
248
249				/* MOVE to next lock */
250				sze->ct_rx_lck = sze->ct_rx_lck->next;
251				sze->ct_rx_cur_ptr =
252					(void *)(((uint8_t *)
253					(sze->ct_rx_lck->start)) + rem_size);
254
255				packet_ptr1 = sze->ct_rx_cur_ptr;
256				packet_len1 = packet_size;
257				packet_ptr2 = NULL;
258				packet_len2 = 0;
259
260				sze->ct_rx_cur_ptr +=
261					RTE_SZE2_ALIGN8(packet_size);
262				sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
263					rem_size - RTE_SZE2_ALIGN8(packet_size);
264			} else {
265				/* get pointer and length from first part */
266				packet_ptr1 = sze->ct_rx_cur_ptr +
267					hw_data_align;
268				packet_len1 = sze->ct_rx_rem_bytes -
269					hw_data_align;
270
271				/* MOVE to next lock */
272				sze->ct_rx_lck = sze->ct_rx_lck->next;
273				sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
274
275				/* get pointer and length from second part */
276				packet_ptr2 = sze->ct_rx_cur_ptr;
277				packet_len2 = packet_size - packet_len1;
278
279				sze->ct_rx_cur_ptr +=
280					RTE_SZE2_ALIGN8(packet_size) -
281					packet_len1;
282				sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
283					(RTE_SZE2_ALIGN8(packet_size) -
284					 packet_len1);
285			}
286		}
287
288		if (unlikely(packet_ptr1 == NULL)) {
289			rte_pktmbuf_free(mbuf);
290			break;
291		}
292
293		/* get the space available for data in the mbuf */
294		mbp_priv = rte_mempool_get_priv(sze_q->mb_pool);
295		buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
296				RTE_PKTMBUF_HEADROOM);
297
298		if (packet_size <= buf_size) {
299			/* sze packet will fit in one mbuf, go ahead and copy */
300			rte_memcpy(rte_pktmbuf_mtod(mbuf, void *),
301					packet_ptr1, packet_len1);
302			if (packet_ptr2 != NULL) {
303				rte_memcpy((void *)(rte_pktmbuf_mtod(mbuf,
304					uint8_t *) + packet_len1),
305					packet_ptr2, packet_len2);
306			}
307			mbuf->data_len = (uint16_t)packet_size;
308
309			mbuf->pkt_len = packet_size;
310			mbuf->port = sze_q->in_port;
311			bufs[num_rx] = mbuf;
312			num_rx++;
313			num_bytes += packet_size;
314		} else {
315			/*
316			 * sze packet will not fit in one mbuf,
317			 * scattered mode is not enabled, drop packet
318			 */
319			RTE_LOG(ERR, PMD,
320				"SZE segment %d bytes will not fit in one mbuf "
321				"(%d bytes), scattered mode is not enabled, "
322				"drop packet!!\n",
323				packet_size, buf_size);
324			rte_pktmbuf_free(mbuf);
325		}
326	}
327
328	sze_q->rx_pkts += num_rx;
329	sze_q->rx_bytes += num_bytes;
330	return num_rx;
331}
332
333static uint16_t
334eth_szedata2_rx_scattered(void *queue,
335		struct rte_mbuf **bufs,
336		uint16_t nb_pkts)
337{
338	unsigned int i;
339	struct rte_mbuf *mbuf;
340	struct szedata2_rx_queue *sze_q = queue;
341	struct rte_pktmbuf_pool_private *mbp_priv;
342	uint16_t num_rx = 0;
343	uint16_t buf_size;
344	uint16_t sg_size;
345	uint16_t hw_size;
346	uint16_t packet_size;
347	uint64_t num_bytes = 0;
348	struct szedata *sze = sze_q->sze;
349	uint8_t *header_ptr = NULL; /* header of packet */
350	uint8_t *packet_ptr1 = NULL;
351	uint8_t *packet_ptr2 = NULL;
352	uint16_t packet_len1 = 0;
353	uint16_t packet_len2 = 0;
354	uint16_t hw_data_align;
355
356	if (unlikely(sze_q->sze == NULL || nb_pkts == 0))
357		return 0;
358
359	/*
360	 * Reads the given number of packets from szedata2 channel given
361	 * by queue and copies the packet data into a newly allocated mbuf
362	 * to return.
363	 */
364	for (i = 0; i < nb_pkts; i++) {
365		const struct szedata_lock *ct_rx_lck_backup;
366		unsigned int ct_rx_rem_bytes_backup;
367		unsigned char *ct_rx_cur_ptr_backup;
368
369		/* get the next sze packet */
370		if (sze->ct_rx_lck != NULL && !sze->ct_rx_rem_bytes &&
371				sze->ct_rx_lck->next == NULL) {
372			/* unlock old data */
373			szedata_rx_unlock_data(sze_q->sze, sze->ct_rx_lck_orig);
374			sze->ct_rx_lck_orig = NULL;
375			sze->ct_rx_lck = NULL;
376		}
377
378		/*
379		 * Store items from sze structure which can be changed
380		 * before mbuf allocating. Use these items in case of mbuf
381		 * allocating failure.
382		 */
383		ct_rx_lck_backup = sze->ct_rx_lck;
384		ct_rx_rem_bytes_backup = sze->ct_rx_rem_bytes;
385		ct_rx_cur_ptr_backup = sze->ct_rx_cur_ptr;
386
387		if (!sze->ct_rx_rem_bytes && sze->ct_rx_lck_orig == NULL) {
388			/* nothing to read, lock new data */
389			sze->ct_rx_lck = szedata_rx_lock_data(sze_q->sze, ~0U);
390			sze->ct_rx_lck_orig = sze->ct_rx_lck;
391
392			/*
393			 * Backup items from sze structure must be updated
394			 * after locking to contain pointers to new locks.
395			 */
396			ct_rx_lck_backup = sze->ct_rx_lck;
397			ct_rx_rem_bytes_backup = sze->ct_rx_rem_bytes;
398			ct_rx_cur_ptr_backup = sze->ct_rx_cur_ptr;
399
400			if (sze->ct_rx_lck == NULL)
401				/* nothing to lock */
402				break;
403
404			sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
405			sze->ct_rx_rem_bytes = sze->ct_rx_lck->len;
406
407			if (!sze->ct_rx_rem_bytes)
408				break;
409		}
410
411		if (sze->ct_rx_rem_bytes < RTE_SZE2_PACKET_HEADER_SIZE) {
412			/*
413			 * cut in header - copy parts of header to merge buffer
414			 */
415			if (sze->ct_rx_lck->next == NULL)
416				break;
417
418			/* copy first part of header */
419			rte_memcpy(sze->ct_rx_buffer, sze->ct_rx_cur_ptr,
420					sze->ct_rx_rem_bytes);
421
422			/* copy second part of header */
423			sze->ct_rx_lck = sze->ct_rx_lck->next;
424			sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
425			rte_memcpy(sze->ct_rx_buffer + sze->ct_rx_rem_bytes,
426				sze->ct_rx_cur_ptr,
427				RTE_SZE2_PACKET_HEADER_SIZE -
428				sze->ct_rx_rem_bytes);
429
430			sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE -
431				sze->ct_rx_rem_bytes;
432			sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
433				RTE_SZE2_PACKET_HEADER_SIZE +
434				sze->ct_rx_rem_bytes;
435
436			header_ptr = (uint8_t *)sze->ct_rx_buffer;
437		} else {
438			/* not cut */
439			header_ptr = (uint8_t *)sze->ct_rx_cur_ptr;
440			sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE;
441			sze->ct_rx_rem_bytes -= RTE_SZE2_PACKET_HEADER_SIZE;
442		}
443
444		sg_size = le16toh(*((uint16_t *)header_ptr));
445		hw_size = le16toh(*(((uint16_t *)header_ptr) + 1));
446		packet_size = sg_size -
447			RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size);
448
449
450		/* checks if packet all right */
451		if (!sg_size)
452			errx(5, "Zero segsize");
453
454		/* check sg_size and hwsize */
455		if (hw_size > sg_size - RTE_SZE2_PACKET_HEADER_SIZE) {
456			errx(10, "Hwsize bigger than expected. Segsize: %d, "
457					"hwsize: %d", sg_size, hw_size);
458		}
459
460		hw_data_align =
461			RTE_SZE2_ALIGN8((RTE_SZE2_PACKET_HEADER_SIZE +
462			hw_size)) - RTE_SZE2_PACKET_HEADER_SIZE;
463
464		if (sze->ct_rx_rem_bytes >=
465				(uint16_t)(sg_size -
466				RTE_SZE2_PACKET_HEADER_SIZE)) {
467			/* no cut */
468			/* one packet ready - go to another */
469			packet_ptr1 = sze->ct_rx_cur_ptr + hw_data_align;
470			packet_len1 = packet_size;
471			packet_ptr2 = NULL;
472			packet_len2 = 0;
473
474			sze->ct_rx_cur_ptr += RTE_SZE2_ALIGN8(sg_size) -
475				RTE_SZE2_PACKET_HEADER_SIZE;
476			sze->ct_rx_rem_bytes -= RTE_SZE2_ALIGN8(sg_size) -
477				RTE_SZE2_PACKET_HEADER_SIZE;
478		} else {
479			/* cut in data */
480			if (sze->ct_rx_lck->next == NULL) {
481				errx(6, "Need \"next\" lock, but it is "
482					"missing: %u", sze->ct_rx_rem_bytes);
483			}
484
485			/* skip hw data */
486			if (sze->ct_rx_rem_bytes <= hw_data_align) {
487				uint16_t rem_size = hw_data_align -
488					sze->ct_rx_rem_bytes;
489
490				/* MOVE to next lock */
491				sze->ct_rx_lck = sze->ct_rx_lck->next;
492				sze->ct_rx_cur_ptr =
493					(void *)(((uint8_t *)
494					(sze->ct_rx_lck->start)) + rem_size);
495
496				packet_ptr1 = sze->ct_rx_cur_ptr;
497				packet_len1 = packet_size;
498				packet_ptr2 = NULL;
499				packet_len2 = 0;
500
501				sze->ct_rx_cur_ptr +=
502					RTE_SZE2_ALIGN8(packet_size);
503				sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
504					rem_size - RTE_SZE2_ALIGN8(packet_size);
505			} else {
506				/* get pointer and length from first part */
507				packet_ptr1 = sze->ct_rx_cur_ptr +
508					hw_data_align;
509				packet_len1 = sze->ct_rx_rem_bytes -
510					hw_data_align;
511
512				/* MOVE to next lock */
513				sze->ct_rx_lck = sze->ct_rx_lck->next;
514				sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
515
516				/* get pointer and length from second part */
517				packet_ptr2 = sze->ct_rx_cur_ptr;
518				packet_len2 = packet_size - packet_len1;
519
520				sze->ct_rx_cur_ptr +=
521					RTE_SZE2_ALIGN8(packet_size) -
522					packet_len1;
523				sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
524					(RTE_SZE2_ALIGN8(packet_size) -
525					 packet_len1);
526			}
527		}
528
529		if (unlikely(packet_ptr1 == NULL))
530			break;
531
532		mbuf = rte_pktmbuf_alloc(sze_q->mb_pool);
533
534		if (unlikely(mbuf == NULL)) {
535			/*
536			 * Restore items from sze structure to state after
537			 * unlocking (eventually locking).
538			 */
539			sze->ct_rx_lck = ct_rx_lck_backup;
540			sze->ct_rx_rem_bytes = ct_rx_rem_bytes_backup;
541			sze->ct_rx_cur_ptr = ct_rx_cur_ptr_backup;
542			break;
543		}
544
545		/* get the space available for data in the mbuf */
546		mbp_priv = rte_mempool_get_priv(sze_q->mb_pool);
547		buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
548				RTE_PKTMBUF_HEADROOM);
549
550		if (packet_size <= buf_size) {
551			/* sze packet will fit in one mbuf, go ahead and copy */
552			rte_memcpy(rte_pktmbuf_mtod(mbuf, void *),
553					packet_ptr1, packet_len1);
554			if (packet_ptr2 != NULL) {
555				rte_memcpy((void *)
556					(rte_pktmbuf_mtod(mbuf, uint8_t *) +
557					packet_len1), packet_ptr2, packet_len2);
558			}
559			mbuf->data_len = (uint16_t)packet_size;
560		} else {
561			/*
562			 * sze packet will not fit in one mbuf,
563			 * scatter packet into more mbufs
564			 */
565			struct rte_mbuf *m = mbuf;
566			uint16_t len = rte_pktmbuf_tailroom(mbuf);
567
568			/* copy first part of packet */
569			/* fill first mbuf */
570			rte_memcpy(rte_pktmbuf_append(mbuf, len), packet_ptr1,
571				len);
572			packet_len1 -= len;
573			packet_ptr1 = ((uint8_t *)packet_ptr1) + len;
574
575			while (packet_len1 > 0) {
576				/* fill new mbufs */
577				m->next = rte_pktmbuf_alloc(sze_q->mb_pool);
578
579				if (unlikely(m->next == NULL)) {
580					rte_pktmbuf_free(mbuf);
581					/*
582					 * Restore items from sze structure
583					 * to state after unlocking (eventually
584					 * locking).
585					 */
586					sze->ct_rx_lck = ct_rx_lck_backup;
587					sze->ct_rx_rem_bytes =
588						ct_rx_rem_bytes_backup;
589					sze->ct_rx_cur_ptr =
590						ct_rx_cur_ptr_backup;
591					goto finish;
592				}
593
594				m = m->next;
595
596				len = RTE_MIN(rte_pktmbuf_tailroom(m),
597					packet_len1);
598				rte_memcpy(rte_pktmbuf_append(mbuf, len),
599					packet_ptr1, len);
600
601				(mbuf->nb_segs)++;
602				packet_len1 -= len;
603				packet_ptr1 = ((uint8_t *)packet_ptr1) + len;
604			}
605
606			if (packet_ptr2 != NULL) {
607				/* copy second part of packet, if exists */
608				/* fill the rest of currently last mbuf */
609				len = rte_pktmbuf_tailroom(m);
610				rte_memcpy(rte_pktmbuf_append(mbuf, len),
611					packet_ptr2, len);
612				packet_len2 -= len;
613				packet_ptr2 = ((uint8_t *)packet_ptr2) + len;
614
615				while (packet_len2 > 0) {
616					/* fill new mbufs */
617					m->next = rte_pktmbuf_alloc(
618							sze_q->mb_pool);
619
620					if (unlikely(m->next == NULL)) {
621						rte_pktmbuf_free(mbuf);
622						/*
623						 * Restore items from sze
624						 * structure to state after
625						 * unlocking (eventually
626						 * locking).
627						 */
628						sze->ct_rx_lck =
629							ct_rx_lck_backup;
630						sze->ct_rx_rem_bytes =
631							ct_rx_rem_bytes_backup;
632						sze->ct_rx_cur_ptr =
633							ct_rx_cur_ptr_backup;
634						goto finish;
635					}
636
637					m = m->next;
638
639					len = RTE_MIN(rte_pktmbuf_tailroom(m),
640						packet_len2);
641					rte_memcpy(
642						rte_pktmbuf_append(mbuf, len),
643						packet_ptr2, len);
644
645					(mbuf->nb_segs)++;
646					packet_len2 -= len;
647					packet_ptr2 = ((uint8_t *)packet_ptr2) +
648						len;
649				}
650			}
651		}
652		mbuf->pkt_len = packet_size;
653		mbuf->port = sze_q->in_port;
654		bufs[num_rx] = mbuf;
655		num_rx++;
656		num_bytes += packet_size;
657	}
658
659finish:
660	sze_q->rx_pkts += num_rx;
661	sze_q->rx_bytes += num_bytes;
662	return num_rx;
663}
664
665static uint16_t
666eth_szedata2_tx(void *queue,
667		struct rte_mbuf **bufs,
668		uint16_t nb_pkts)
669{
670	struct rte_mbuf *mbuf;
671	struct szedata2_tx_queue *sze_q = queue;
672	uint16_t num_tx = 0;
673	uint64_t num_bytes = 0;
674
675	const struct szedata_lock *lck;
676	uint32_t lock_size;
677	uint32_t lock_size2;
678	void *dst;
679	uint32_t pkt_len;
680	uint32_t hwpkt_len;
681	uint32_t unlock_size;
682	uint32_t rem_len;
683	uint8_t mbuf_segs;
684	uint16_t pkt_left = nb_pkts;
685
686	if (sze_q->sze == NULL || nb_pkts == 0)
687		return 0;
688
689	while (pkt_left > 0) {
690		unlock_size = 0;
691		lck = szedata_tx_lock_data(sze_q->sze,
692			RTE_ETH_SZEDATA2_TX_LOCK_SIZE,
693			sze_q->tx_channel);
694		if (lck == NULL)
695			continue;
696
697		dst = lck->start;
698		lock_size = lck->len;
699		lock_size2 = lck->next ? lck->next->len : 0;
700
701next_packet:
702		mbuf = bufs[nb_pkts - pkt_left];
703
704		pkt_len = mbuf->pkt_len;
705		mbuf_segs = mbuf->nb_segs;
706
707		hwpkt_len = RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
708			RTE_SZE2_ALIGN8(pkt_len);
709
710		if (lock_size + lock_size2 < hwpkt_len) {
711			szedata_tx_unlock_data(sze_q->sze, lck, unlock_size);
712			continue;
713		}
714
715		num_bytes += pkt_len;
716
717		if (lock_size > hwpkt_len) {
718			void *tmp_dst;
719
720			rem_len = 0;
721
722			/* write packet length at first 2 bytes in 8B header */
723			*((uint16_t *)dst) = htole16(
724					RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
725					pkt_len);
726			*(((uint16_t *)dst) + 1) = htole16(0);
727
728			/* copy packet from mbuf */
729			tmp_dst = ((uint8_t *)(dst)) +
730				RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
731			if (mbuf_segs == 1) {
732				/*
733				 * non-scattered packet,
734				 * transmit from one mbuf
735				 */
736				rte_memcpy(tmp_dst,
737					rte_pktmbuf_mtod(mbuf, const void *),
738					pkt_len);
739			} else {
740				/* scattered packet, transmit from more mbufs */
741				struct rte_mbuf *m = mbuf;
742				while (m) {
743					rte_memcpy(tmp_dst,
744						rte_pktmbuf_mtod(m,
745						const void *),
746						m->data_len);
747					tmp_dst = ((uint8_t *)(tmp_dst)) +
748						m->data_len;
749					m = m->next;
750				}
751			}
752
753
754			dst = ((uint8_t *)dst) + hwpkt_len;
755			unlock_size += hwpkt_len;
756			lock_size -= hwpkt_len;
757
758			rte_pktmbuf_free(mbuf);
759			num_tx++;
760			pkt_left--;
761			if (pkt_left == 0) {
762				szedata_tx_unlock_data(sze_q->sze, lck,
763					unlock_size);
764				break;
765			}
766			goto next_packet;
767		} else if (lock_size + lock_size2 >= hwpkt_len) {
768			void *tmp_dst;
769			uint16_t write_len;
770
771			/* write packet length at first 2 bytes in 8B header */
772			*((uint16_t *)dst) =
773				htole16(RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
774					pkt_len);
775			*(((uint16_t *)dst) + 1) = htole16(0);
776
777			/*
778			 * If the raw packet (pkt_len) is smaller than lock_size
779			 * get the correct length for memcpy
780			 */
781			write_len =
782				pkt_len < lock_size -
783				RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED ?
784				pkt_len :
785				lock_size - RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
786
787			rem_len = hwpkt_len - lock_size;
788
789			tmp_dst = ((uint8_t *)(dst)) +
790				RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
791			if (mbuf_segs == 1) {
792				/*
793				 * non-scattered packet,
794				 * transmit from one mbuf
795				 */
796				/* copy part of packet to first area */
797				rte_memcpy(tmp_dst,
798					rte_pktmbuf_mtod(mbuf, const void *),
799					write_len);
800
801				if (lck->next)
802					dst = lck->next->start;
803
804				/* copy part of packet to second area */
805				rte_memcpy(dst,
806					(const void *)(rte_pktmbuf_mtod(mbuf,
807							const uint8_t *) +
808					write_len), pkt_len - write_len);
809			} else {
810				/* scattered packet, transmit from more mbufs */
811				struct rte_mbuf *m = mbuf;
812				uint16_t written = 0;
813				uint16_t to_write = 0;
814				bool new_mbuf = true;
815				uint16_t write_off = 0;
816
817				/* copy part of packet to first area */
818				while (m && written < write_len) {
819					to_write = RTE_MIN(m->data_len,
820							write_len - written);
821					rte_memcpy(tmp_dst,
822						rte_pktmbuf_mtod(m,
823							const void *),
824						to_write);
825
826					tmp_dst = ((uint8_t *)(tmp_dst)) +
827						to_write;
828					if (m->data_len <= write_len -
829							written) {
830						m = m->next;
831						new_mbuf = true;
832					} else {
833						new_mbuf = false;
834					}
835					written += to_write;
836				}
837
838				if (lck->next)
839					dst = lck->next->start;
840
841				tmp_dst = dst;
842				written = 0;
843				write_off = new_mbuf ? 0 : to_write;
844
845				/* copy part of packet to second area */
846				while (m && written < pkt_len - write_len) {
847					rte_memcpy(tmp_dst, (const void *)
848						(rte_pktmbuf_mtod(m,
849						uint8_t *) + write_off),
850						m->data_len - write_off);
851
852					tmp_dst = ((uint8_t *)(tmp_dst)) +
853						(m->data_len - write_off);
854					written += m->data_len - write_off;
855					m = m->next;
856					write_off = 0;
857				}
858			}
859
860			dst = ((uint8_t *)dst) + rem_len;
861			unlock_size += hwpkt_len;
862			lock_size = lock_size2 - rem_len;
863			lock_size2 = 0;
864
865			rte_pktmbuf_free(mbuf);
866			num_tx++;
867		}
868
869		szedata_tx_unlock_data(sze_q->sze, lck, unlock_size);
870		pkt_left--;
871	}
872
873	sze_q->tx_pkts += num_tx;
874	sze_q->err_pkts += nb_pkts - num_tx;
875	sze_q->tx_bytes += num_bytes;
876	return num_tx;
877}
878
879static int
880eth_rx_queue_start(struct rte_eth_dev *dev, uint16_t rxq_id)
881{
882	struct szedata2_rx_queue *rxq = dev->data->rx_queues[rxq_id];
883	int ret;
884	struct pmd_internals *internals = (struct pmd_internals *)
885		dev->data->dev_private;
886
887	if (rxq->sze == NULL) {
888		uint32_t rx = 1 << rxq->rx_channel;
889		uint32_t tx = 0;
890		rxq->sze = szedata_open(internals->sze_dev);
891		if (rxq->sze == NULL)
892			return -EINVAL;
893		ret = szedata_subscribe3(rxq->sze, &rx, &tx);
894		if (ret != 0 || rx == 0)
895			goto err;
896	}
897
898	ret = szedata_start(rxq->sze);
899	if (ret != 0)
900		goto err;
901	dev->data->rx_queue_state[rxq_id] = RTE_ETH_QUEUE_STATE_STARTED;
902	return 0;
903
904err:
905	szedata_close(rxq->sze);
906	rxq->sze = NULL;
907	return -EINVAL;
908}
909
910static int
911eth_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rxq_id)
912{
913	struct szedata2_rx_queue *rxq = dev->data->rx_queues[rxq_id];
914
915	if (rxq->sze != NULL) {
916		szedata_close(rxq->sze);
917		rxq->sze = NULL;
918	}
919
920	dev->data->rx_queue_state[rxq_id] = RTE_ETH_QUEUE_STATE_STOPPED;
921	return 0;
922}
923
924static int
925eth_tx_queue_start(struct rte_eth_dev *dev, uint16_t txq_id)
926{
927	struct szedata2_tx_queue *txq = dev->data->tx_queues[txq_id];
928	int ret;
929	struct pmd_internals *internals = (struct pmd_internals *)
930		dev->data->dev_private;
931
932	if (txq->sze == NULL) {
933		uint32_t rx = 0;
934		uint32_t tx = 1 << txq->tx_channel;
935		txq->sze = szedata_open(internals->sze_dev);
936		if (txq->sze == NULL)
937			return -EINVAL;
938		ret = szedata_subscribe3(txq->sze, &rx, &tx);
939		if (ret != 0 || tx == 0)
940			goto err;
941	}
942
943	ret = szedata_start(txq->sze);
944	if (ret != 0)
945		goto err;
946	dev->data->tx_queue_state[txq_id] = RTE_ETH_QUEUE_STATE_STARTED;
947	return 0;
948
949err:
950	szedata_close(txq->sze);
951	txq->sze = NULL;
952	return -EINVAL;
953}
954
955static int
956eth_tx_queue_stop(struct rte_eth_dev *dev, uint16_t txq_id)
957{
958	struct szedata2_tx_queue *txq = dev->data->tx_queues[txq_id];
959
960	if (txq->sze != NULL) {
961		szedata_close(txq->sze);
962		txq->sze = NULL;
963	}
964
965	dev->data->tx_queue_state[txq_id] = RTE_ETH_QUEUE_STATE_STOPPED;
966	return 0;
967}
968
969static int
970eth_dev_start(struct rte_eth_dev *dev)
971{
972	int ret;
973	uint16_t i;
974	uint16_t nb_rx = dev->data->nb_rx_queues;
975	uint16_t nb_tx = dev->data->nb_tx_queues;
976
977	for (i = 0; i < nb_rx; i++) {
978		ret = eth_rx_queue_start(dev, i);
979		if (ret != 0)
980			goto err_rx;
981	}
982
983	for (i = 0; i < nb_tx; i++) {
984		ret = eth_tx_queue_start(dev, i);
985		if (ret != 0)
986			goto err_tx;
987	}
988
989	return 0;
990
991err_tx:
992	for (i = 0; i < nb_tx; i++)
993		eth_tx_queue_stop(dev, i);
994err_rx:
995	for (i = 0; i < nb_rx; i++)
996		eth_rx_queue_stop(dev, i);
997	return ret;
998}
999
1000static void
1001eth_dev_stop(struct rte_eth_dev *dev)
1002{
1003	uint16_t i;
1004	uint16_t nb_rx = dev->data->nb_rx_queues;
1005	uint16_t nb_tx = dev->data->nb_tx_queues;
1006
1007	for (i = 0; i < nb_tx; i++)
1008		eth_tx_queue_stop(dev, i);
1009
1010	for (i = 0; i < nb_rx; i++)
1011		eth_rx_queue_stop(dev, i);
1012}
1013
1014static int
1015eth_dev_configure(struct rte_eth_dev *dev)
1016{
1017	struct rte_eth_dev_data *data = dev->data;
1018	if (data->dev_conf.rxmode.enable_scatter == 1) {
1019		dev->rx_pkt_burst = eth_szedata2_rx_scattered;
1020		data->scattered_rx = 1;
1021	} else {
1022		dev->rx_pkt_burst = eth_szedata2_rx;
1023		data->scattered_rx = 0;
1024	}
1025	return 0;
1026}
1027
1028static void
1029eth_dev_info(struct rte_eth_dev *dev,
1030		struct rte_eth_dev_info *dev_info)
1031{
1032	struct pmd_internals *internals = dev->data->dev_private;
1033	dev_info->if_index = 0;
1034	dev_info->max_mac_addrs = 1;
1035	dev_info->max_rx_pktlen = (uint32_t)-1;
1036	dev_info->max_rx_queues = internals->max_rx_queues;
1037	dev_info->max_tx_queues = internals->max_tx_queues;
1038	dev_info->min_rx_bufsize = 0;
1039	dev_info->speed_capa = ETH_LINK_SPEED_100G;
1040}
1041
1042static void
1043eth_stats_get(struct rte_eth_dev *dev,
1044		struct rte_eth_stats *stats)
1045{
1046	uint16_t i;
1047	uint16_t nb_rx = dev->data->nb_rx_queues;
1048	uint16_t nb_tx = dev->data->nb_tx_queues;
1049	uint64_t rx_total = 0;
1050	uint64_t tx_total = 0;
1051	uint64_t tx_err_total = 0;
1052	uint64_t rx_total_bytes = 0;
1053	uint64_t tx_total_bytes = 0;
1054
1055	for (i = 0; i < nb_rx; i++) {
1056		struct szedata2_rx_queue *rxq = dev->data->rx_queues[i];
1057
1058		if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
1059			stats->q_ipackets[i] = rxq->rx_pkts;
1060			stats->q_ibytes[i] = rxq->rx_bytes;
1061		}
1062		rx_total += rxq->rx_pkts;
1063		rx_total_bytes += rxq->rx_bytes;
1064	}
1065
1066	for (i = 0; i < nb_tx; i++) {
1067		struct szedata2_tx_queue *txq = dev->data->tx_queues[i];
1068
1069		if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
1070			stats->q_opackets[i] = txq->tx_pkts;
1071			stats->q_obytes[i] = txq->tx_bytes;
1072			stats->q_errors[i] = txq->err_pkts;
1073		}
1074		tx_total += txq->tx_pkts;
1075		tx_total_bytes += txq->tx_bytes;
1076		tx_err_total += txq->err_pkts;
1077	}
1078
1079	stats->ipackets = rx_total;
1080	stats->opackets = tx_total;
1081	stats->ibytes = rx_total_bytes;
1082	stats->obytes = tx_total_bytes;
1083	stats->oerrors = tx_err_total;
1084}
1085
1086static void
1087eth_stats_reset(struct rte_eth_dev *dev)
1088{
1089	uint16_t i;
1090	uint16_t nb_rx = dev->data->nb_rx_queues;
1091	uint16_t nb_tx = dev->data->nb_tx_queues;
1092	struct pmd_internals *internals = dev->data->dev_private;
1093
1094	for (i = 0; i < nb_rx; i++) {
1095		internals->rx_queue[i].rx_pkts = 0;
1096		internals->rx_queue[i].rx_bytes = 0;
1097		internals->rx_queue[i].err_pkts = 0;
1098	}
1099	for (i = 0; i < nb_tx; i++) {
1100		internals->tx_queue[i].tx_pkts = 0;
1101		internals->tx_queue[i].tx_bytes = 0;
1102		internals->tx_queue[i].err_pkts = 0;
1103	}
1104}
1105
1106static void
1107eth_rx_queue_release(void *q)
1108{
1109	struct szedata2_rx_queue *rxq = (struct szedata2_rx_queue *)q;
1110	if (rxq->sze != NULL) {
1111		szedata_close(rxq->sze);
1112		rxq->sze = NULL;
1113	}
1114}
1115
1116static void
1117eth_tx_queue_release(void *q)
1118{
1119	struct szedata2_tx_queue *txq = (struct szedata2_tx_queue *)q;
1120	if (txq->sze != NULL) {
1121		szedata_close(txq->sze);
1122		txq->sze = NULL;
1123	}
1124}
1125
1126static void
1127eth_dev_close(struct rte_eth_dev *dev)
1128{
1129	uint16_t i;
1130	uint16_t nb_rx = dev->data->nb_rx_queues;
1131	uint16_t nb_tx = dev->data->nb_tx_queues;
1132
1133	eth_dev_stop(dev);
1134
1135	for (i = 0; i < nb_rx; i++) {
1136		eth_rx_queue_release(dev->data->rx_queues[i]);
1137		dev->data->rx_queues[i] = NULL;
1138	}
1139	dev->data->nb_rx_queues = 0;
1140	for (i = 0; i < nb_tx; i++) {
1141		eth_tx_queue_release(dev->data->tx_queues[i]);
1142		dev->data->tx_queues[i] = NULL;
1143	}
1144	dev->data->nb_tx_queues = 0;
1145}
1146
1147static int
1148eth_link_update(struct rte_eth_dev *dev,
1149		int wait_to_complete __rte_unused)
1150{
1151	struct rte_eth_link link;
1152	struct rte_eth_link *link_ptr = &link;
1153	struct rte_eth_link *dev_link = &dev->data->dev_link;
1154	volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1155			dev, SZEDATA2_CGMII_IBUF_BASE_OFF,
1156			volatile struct szedata2_cgmii_ibuf *);
1157
1158	switch (cgmii_link_speed(ibuf)) {
1159	case SZEDATA2_LINK_SPEED_10G:
1160		link.link_speed = ETH_SPEED_NUM_10G;
1161		break;
1162	case SZEDATA2_LINK_SPEED_40G:
1163		link.link_speed = ETH_SPEED_NUM_40G;
1164		break;
1165	case SZEDATA2_LINK_SPEED_100G:
1166		link.link_speed = ETH_SPEED_NUM_100G;
1167		break;
1168	default:
1169		link.link_speed = ETH_SPEED_NUM_10G;
1170		break;
1171	}
1172
1173	/* szedata2 uses only full duplex */
1174	link.link_duplex = ETH_LINK_FULL_DUPLEX;
1175
1176	link.link_status = (cgmii_ibuf_is_enabled(ibuf) &&
1177			cgmii_ibuf_is_link_up(ibuf)) ? ETH_LINK_UP : ETH_LINK_DOWN;
1178
1179	link.link_autoneg = ETH_LINK_FIXED;
1180
1181	rte_atomic64_cmpset((uint64_t *)dev_link, *(uint64_t *)dev_link,
1182			*(uint64_t *)link_ptr);
1183
1184	return 0;
1185}
1186
1187static int
1188eth_dev_set_link_up(struct rte_eth_dev *dev)
1189{
1190	volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1191			dev, SZEDATA2_CGMII_IBUF_BASE_OFF,
1192			volatile struct szedata2_cgmii_ibuf *);
1193	volatile struct szedata2_cgmii_obuf *obuf = SZEDATA2_PCI_RESOURCE_PTR(
1194			dev, SZEDATA2_CGMII_OBUF_BASE_OFF,
1195			volatile struct szedata2_cgmii_obuf *);
1196
1197	cgmii_ibuf_enable(ibuf);
1198	cgmii_obuf_enable(obuf);
1199	return 0;
1200}
1201
1202static int
1203eth_dev_set_link_down(struct rte_eth_dev *dev)
1204{
1205	volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1206			dev, SZEDATA2_CGMII_IBUF_BASE_OFF,
1207			volatile struct szedata2_cgmii_ibuf *);
1208	volatile struct szedata2_cgmii_obuf *obuf = SZEDATA2_PCI_RESOURCE_PTR(
1209			dev, SZEDATA2_CGMII_OBUF_BASE_OFF,
1210			volatile struct szedata2_cgmii_obuf *);
1211
1212	cgmii_ibuf_disable(ibuf);
1213	cgmii_obuf_disable(obuf);
1214	return 0;
1215}
1216
1217static int
1218eth_rx_queue_setup(struct rte_eth_dev *dev,
1219		uint16_t rx_queue_id,
1220		uint16_t nb_rx_desc __rte_unused,
1221		unsigned int socket_id __rte_unused,
1222		const struct rte_eth_rxconf *rx_conf __rte_unused,
1223		struct rte_mempool *mb_pool)
1224{
1225	struct pmd_internals *internals = dev->data->dev_private;
1226	struct szedata2_rx_queue *rxq = &internals->rx_queue[rx_queue_id];
1227	int ret;
1228	uint32_t rx = 1 << rx_queue_id;
1229	uint32_t tx = 0;
1230
1231	rxq->sze = szedata_open(internals->sze_dev);
1232	if (rxq->sze == NULL)
1233		return -EINVAL;
1234	ret = szedata_subscribe3(rxq->sze, &rx, &tx);
1235	if (ret != 0 || rx == 0) {
1236		szedata_close(rxq->sze);
1237		rxq->sze = NULL;
1238		return -EINVAL;
1239	}
1240	rxq->rx_channel = rx_queue_id;
1241	rxq->in_port = dev->data->port_id;
1242	rxq->mb_pool = mb_pool;
1243	rxq->rx_pkts = 0;
1244	rxq->rx_bytes = 0;
1245	rxq->err_pkts = 0;
1246
1247	dev->data->rx_queues[rx_queue_id] = rxq;
1248	return 0;
1249}
1250
1251static int
1252eth_tx_queue_setup(struct rte_eth_dev *dev,
1253		uint16_t tx_queue_id,
1254		uint16_t nb_tx_desc __rte_unused,
1255		unsigned int socket_id __rte_unused,
1256		const struct rte_eth_txconf *tx_conf __rte_unused)
1257{
1258	struct pmd_internals *internals = dev->data->dev_private;
1259	struct szedata2_tx_queue *txq = &internals->tx_queue[tx_queue_id];
1260	int ret;
1261	uint32_t rx = 0;
1262	uint32_t tx = 1 << tx_queue_id;
1263
1264	txq->sze = szedata_open(internals->sze_dev);
1265	if (txq->sze == NULL)
1266		return -EINVAL;
1267	ret = szedata_subscribe3(txq->sze, &rx, &tx);
1268	if (ret != 0 || tx == 0) {
1269		szedata_close(txq->sze);
1270		txq->sze = NULL;
1271		return -EINVAL;
1272	}
1273	txq->tx_channel = tx_queue_id;
1274	txq->tx_pkts = 0;
1275	txq->tx_bytes = 0;
1276	txq->err_pkts = 0;
1277
1278	dev->data->tx_queues[tx_queue_id] = txq;
1279	return 0;
1280}
1281
1282static void
1283eth_mac_addr_set(struct rte_eth_dev *dev __rte_unused,
1284		struct ether_addr *mac_addr __rte_unused)
1285{
1286}
1287
1288static void
1289eth_promiscuous_enable(struct rte_eth_dev *dev)
1290{
1291	volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1292			dev, SZEDATA2_CGMII_IBUF_BASE_OFF,
1293			volatile struct szedata2_cgmii_ibuf *);
1294	cgmii_ibuf_mac_mode_write(ibuf, SZEDATA2_MAC_CHMODE_PROMISC);
1295}
1296
1297static void
1298eth_promiscuous_disable(struct rte_eth_dev *dev)
1299{
1300	volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1301			dev, SZEDATA2_CGMII_IBUF_BASE_OFF,
1302			volatile struct szedata2_cgmii_ibuf *);
1303	cgmii_ibuf_mac_mode_write(ibuf, SZEDATA2_MAC_CHMODE_ONLY_VALID);
1304}
1305
1306static void
1307eth_allmulticast_enable(struct rte_eth_dev *dev)
1308{
1309	volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1310			dev, SZEDATA2_CGMII_IBUF_BASE_OFF,
1311			volatile struct szedata2_cgmii_ibuf *);
1312	cgmii_ibuf_mac_mode_write(ibuf, SZEDATA2_MAC_CHMODE_ALL_MULTICAST);
1313}
1314
1315static void
1316eth_allmulticast_disable(struct rte_eth_dev *dev)
1317{
1318	volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1319			dev, SZEDATA2_CGMII_IBUF_BASE_OFF,
1320			volatile struct szedata2_cgmii_ibuf *);
1321	cgmii_ibuf_mac_mode_write(ibuf, SZEDATA2_MAC_CHMODE_ONLY_VALID);
1322}
1323
1324static const struct eth_dev_ops ops = {
1325	.dev_start          = eth_dev_start,
1326	.dev_stop           = eth_dev_stop,
1327	.dev_set_link_up    = eth_dev_set_link_up,
1328	.dev_set_link_down  = eth_dev_set_link_down,
1329	.dev_close          = eth_dev_close,
1330	.dev_configure      = eth_dev_configure,
1331	.dev_infos_get      = eth_dev_info,
1332	.promiscuous_enable   = eth_promiscuous_enable,
1333	.promiscuous_disable  = eth_promiscuous_disable,
1334	.allmulticast_enable  = eth_allmulticast_enable,
1335	.allmulticast_disable = eth_allmulticast_disable,
1336	.rx_queue_start     = eth_rx_queue_start,
1337	.rx_queue_stop      = eth_rx_queue_stop,
1338	.tx_queue_start     = eth_tx_queue_start,
1339	.tx_queue_stop      = eth_tx_queue_stop,
1340	.rx_queue_setup     = eth_rx_queue_setup,
1341	.tx_queue_setup     = eth_tx_queue_setup,
1342	.rx_queue_release   = eth_rx_queue_release,
1343	.tx_queue_release   = eth_tx_queue_release,
1344	.link_update        = eth_link_update,
1345	.stats_get          = eth_stats_get,
1346	.stats_reset        = eth_stats_reset,
1347	.mac_addr_set       = eth_mac_addr_set,
1348};
1349
1350/*
1351 * This function goes through sysfs and looks for an index of szedata2
1352 * device file (/dev/szedataIIX, where X is the index).
1353 *
1354 * @return
1355 *           0 on success
1356 *          -1 on error
1357 */
1358static int
1359get_szedata2_index(struct rte_eth_dev *dev, uint32_t *index)
1360{
1361	DIR *dir;
1362	struct dirent *entry;
1363	int ret;
1364	uint32_t tmp_index;
1365	FILE *fd;
1366	char pcislot_path[PATH_MAX];
1367	struct rte_pci_addr pcislot_addr = dev->pci_dev->addr;
1368	uint32_t domain;
1369	uint8_t bus;
1370	uint8_t devid;
1371	uint8_t function;
1372
1373	dir = opendir("/sys/class/combo");
1374	if (dir == NULL)
1375		return -1;
1376
1377	/*
1378	 * Iterate through all combosixX directories.
1379	 * When the value in /sys/class/combo/combosixX/device/pcislot
1380	 * file is the location of the ethernet device dev, "X" is the
1381	 * index of the device.
1382	 */
1383	while ((entry = readdir(dir)) != NULL) {
1384		ret = sscanf(entry->d_name, "combosix%u", &tmp_index);
1385		if (ret != 1)
1386			continue;
1387
1388		snprintf(pcislot_path, PATH_MAX,
1389			"/sys/class/combo/combosix%u/device/pcislot",
1390			tmp_index);
1391
1392		fd = fopen(pcislot_path, "r");
1393		if (fd == NULL)
1394			continue;
1395
1396		ret = fscanf(fd, "%8" SCNx32 ":%2" SCNx8 ":%2" SCNx8 ".%" SCNx8,
1397				&domain, &bus, &devid, &function);
1398		fclose(fd);
1399		if (ret != 4)
1400			continue;
1401
1402		if (pcislot_addr.domain == domain &&
1403				pcislot_addr.bus == bus &&
1404				pcislot_addr.devid == devid &&
1405				pcislot_addr.function == function) {
1406			*index = tmp_index;
1407			closedir(dir);
1408			return 0;
1409		}
1410	}
1411
1412	closedir(dir);
1413	return -1;
1414}
1415
1416static int
1417rte_szedata2_eth_dev_init(struct rte_eth_dev *dev)
1418{
1419	struct rte_eth_dev_data *data = dev->data;
1420	struct pmd_internals *internals = (struct pmd_internals *)
1421		data->dev_private;
1422	struct szedata *szedata_temp;
1423	int ret;
1424	uint32_t szedata2_index;
1425	struct rte_pci_addr *pci_addr = &dev->pci_dev->addr;
1426	struct rte_mem_resource *pci_rsc =
1427		&dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER];
1428	char rsc_filename[PATH_MAX];
1429	void *pci_resource_ptr = NULL;
1430	int fd;
1431
1432	RTE_LOG(INFO, PMD, "Initializing szedata2 device (" PCI_PRI_FMT ")\n",
1433			pci_addr->domain, pci_addr->bus, pci_addr->devid,
1434			pci_addr->function);
1435
1436	/* Get index of szedata2 device file and create path to device file */
1437	ret = get_szedata2_index(dev, &szedata2_index);
1438	if (ret != 0) {
1439		RTE_LOG(ERR, PMD, "Failed to get szedata2 device index!\n");
1440		return -ENODEV;
1441	}
1442	snprintf(internals->sze_dev, PATH_MAX, SZEDATA2_DEV_PATH_FMT,
1443			szedata2_index);
1444
1445	RTE_LOG(INFO, PMD, "SZEDATA2 path: %s\n", internals->sze_dev);
1446
1447	/*
1448	 * Get number of available DMA RX and TX channels, which is maximum
1449	 * number of queues that can be created and store it in private device
1450	 * data structure.
1451	 */
1452	szedata_temp = szedata_open(internals->sze_dev);
1453	if (szedata_temp == NULL) {
1454		RTE_LOG(ERR, PMD, "szedata_open(): failed to open %s",
1455				internals->sze_dev);
1456		return -EINVAL;
1457	}
1458	internals->max_rx_queues = szedata_ifaces_available(szedata_temp,
1459			SZE2_DIR_RX);
1460	internals->max_tx_queues = szedata_ifaces_available(szedata_temp,
1461			SZE2_DIR_TX);
1462	szedata_close(szedata_temp);
1463
1464	RTE_LOG(INFO, PMD, "Available DMA channels RX: %u TX: %u\n",
1465			internals->max_rx_queues, internals->max_tx_queues);
1466
1467	/* Set rx, tx burst functions */
1468	if (data->dev_conf.rxmode.enable_scatter == 1 ||
1469		data->scattered_rx == 1) {
1470		dev->rx_pkt_burst = eth_szedata2_rx_scattered;
1471		data->scattered_rx = 1;
1472	} else {
1473		dev->rx_pkt_burst = eth_szedata2_rx;
1474		data->scattered_rx = 0;
1475	}
1476	dev->tx_pkt_burst = eth_szedata2_tx;
1477
1478	/* Set function callbacks for Ethernet API */
1479	dev->dev_ops = &ops;
1480
1481	rte_eth_copy_pci_info(dev, dev->pci_dev);
1482
1483	/* mmap pci resource0 file to rte_mem_resource structure */
1484	if (dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].phys_addr ==
1485			0) {
1486		RTE_LOG(ERR, PMD, "Missing resource%u file\n",
1487				PCI_RESOURCE_NUMBER);
1488		return -EINVAL;
1489	}
1490	snprintf(rsc_filename, PATH_MAX,
1491		"%s/" PCI_PRI_FMT "/resource%u", pci_get_sysfs_path(),
1492		pci_addr->domain, pci_addr->bus,
1493		pci_addr->devid, pci_addr->function, PCI_RESOURCE_NUMBER);
1494	fd = open(rsc_filename, O_RDWR);
1495	if (fd < 0) {
1496		RTE_LOG(ERR, PMD, "Could not open file %s\n", rsc_filename);
1497		return -EINVAL;
1498	}
1499
1500	pci_resource_ptr = mmap(0,
1501			dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len,
1502			PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
1503	close(fd);
1504	if (pci_resource_ptr == MAP_FAILED) {
1505		RTE_LOG(ERR, PMD, "Could not mmap file %s (fd = %d)\n",
1506				rsc_filename, fd);
1507		return -EINVAL;
1508	}
1509	dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr =
1510		pci_resource_ptr;
1511
1512	RTE_LOG(DEBUG, PMD, "resource%u phys_addr = 0x%llx len = %llu "
1513			"virt addr = %llx\n", PCI_RESOURCE_NUMBER,
1514			(unsigned long long)pci_rsc->phys_addr,
1515			(unsigned long long)pci_rsc->len,
1516			(unsigned long long)pci_rsc->addr);
1517
1518	/* Get link state */
1519	eth_link_update(dev, 0);
1520
1521	/* Allocate space for one mac address */
1522	data->mac_addrs = rte_zmalloc(data->name, sizeof(struct ether_addr),
1523			RTE_CACHE_LINE_SIZE);
1524	if (data->mac_addrs == NULL) {
1525		RTE_LOG(ERR, PMD, "Could not alloc space for MAC address!\n");
1526		munmap(dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr,
1527			dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len);
1528		return -EINVAL;
1529	}
1530
1531	ether_addr_copy(&eth_addr, data->mac_addrs);
1532
1533	/* At initial state COMBO card is in promiscuous mode so disable it */
1534	eth_promiscuous_disable(dev);
1535
1536	RTE_LOG(INFO, PMD, "szedata2 device ("
1537			PCI_PRI_FMT ") successfully initialized\n",
1538			pci_addr->domain, pci_addr->bus, pci_addr->devid,
1539			pci_addr->function);
1540
1541	return 0;
1542}
1543
1544static int
1545rte_szedata2_eth_dev_uninit(struct rte_eth_dev *dev)
1546{
1547	struct rte_pci_addr *pci_addr = &dev->pci_dev->addr;
1548
1549	rte_free(dev->data->mac_addrs);
1550	dev->data->mac_addrs = NULL;
1551	munmap(dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr,
1552		dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len);
1553
1554	RTE_LOG(INFO, PMD, "szedata2 device ("
1555			PCI_PRI_FMT ") successfully uninitialized\n",
1556			pci_addr->domain, pci_addr->bus, pci_addr->devid,
1557			pci_addr->function);
1558
1559	return 0;
1560}
1561
1562static const struct rte_pci_id rte_szedata2_pci_id_table[] = {
1563	{
1564		RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1565				PCI_DEVICE_ID_NETCOPE_COMBO80G)
1566	},
1567	{
1568		RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1569				PCI_DEVICE_ID_NETCOPE_COMBO100G)
1570	},
1571	{
1572		RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1573				PCI_DEVICE_ID_NETCOPE_COMBO100G2)
1574	},
1575	{
1576		.vendor_id = 0,
1577	}
1578};
1579
1580static struct eth_driver szedata2_eth_driver = {
1581	.pci_drv = {
1582		.id_table = rte_szedata2_pci_id_table,
1583		.probe = rte_eth_dev_pci_probe,
1584		.remove = rte_eth_dev_pci_remove,
1585	},
1586	.eth_dev_init     = rte_szedata2_eth_dev_init,
1587	.eth_dev_uninit   = rte_szedata2_eth_dev_uninit,
1588	.dev_private_size = sizeof(struct pmd_internals),
1589};
1590
1591RTE_PMD_REGISTER_PCI(RTE_SZEDATA2_DRIVER_NAME, szedata2_eth_driver.pci_drv);
1592RTE_PMD_REGISTER_PCI_TABLE(RTE_SZEDATA2_DRIVER_NAME, rte_szedata2_pci_id_table);
1593