18b25d1adSChristian Ehrhardt/* Copyright 2008-2016 Cisco Systems, Inc.  All rights reserved.
28b25d1adSChristian Ehrhardt * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
38b25d1adSChristian Ehrhardt *
48b25d1adSChristian Ehrhardt * Copyright (c) 2014, Cisco Systems, Inc.
58b25d1adSChristian Ehrhardt * All rights reserved.
68b25d1adSChristian Ehrhardt *
78b25d1adSChristian Ehrhardt * Redistribution and use in source and binary forms, with or without
88b25d1adSChristian Ehrhardt * modification, are permitted provided that the following conditions
98b25d1adSChristian Ehrhardt * are met:
108b25d1adSChristian Ehrhardt *
118b25d1adSChristian Ehrhardt * 1. Redistributions of source code must retain the above copyright
128b25d1adSChristian Ehrhardt * notice, this list of conditions and the following disclaimer.
138b25d1adSChristian Ehrhardt *
148b25d1adSChristian Ehrhardt * 2. Redistributions in binary form must reproduce the above copyright
158b25d1adSChristian Ehrhardt * notice, this list of conditions and the following disclaimer in
168b25d1adSChristian Ehrhardt * the documentation and/or other materials provided with the
178b25d1adSChristian Ehrhardt * distribution.
188b25d1adSChristian Ehrhardt *
198b25d1adSChristian Ehrhardt * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
208b25d1adSChristian Ehrhardt * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
218b25d1adSChristian Ehrhardt * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
228b25d1adSChristian Ehrhardt * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
238b25d1adSChristian Ehrhardt * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
248b25d1adSChristian Ehrhardt * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
258b25d1adSChristian Ehrhardt * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
268b25d1adSChristian Ehrhardt * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
278b25d1adSChristian Ehrhardt * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
288b25d1adSChristian Ehrhardt * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
298b25d1adSChristian Ehrhardt * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
308b25d1adSChristian Ehrhardt * POSSIBILITY OF SUCH DAMAGE.
318b25d1adSChristian Ehrhardt */
328b25d1adSChristian Ehrhardt
338b25d1adSChristian Ehrhardt#include <rte_mbuf.h>
348b25d1adSChristian Ehrhardt#include <rte_ethdev.h>
358b25d1adSChristian Ehrhardt#include <rte_prefetch.h>
368b25d1adSChristian Ehrhardt
378b25d1adSChristian Ehrhardt#include "enic_compat.h"
388b25d1adSChristian Ehrhardt#include "rq_enet_desc.h"
398b25d1adSChristian Ehrhardt#include "enic.h"
408b25d1adSChristian Ehrhardt
418b25d1adSChristian Ehrhardt#define RTE_PMD_USE_PREFETCH
428b25d1adSChristian Ehrhardt
438b25d1adSChristian Ehrhardt#ifdef RTE_PMD_USE_PREFETCH
448b25d1adSChristian Ehrhardt/*Prefetch a cache line into all cache levels. */
458b25d1adSChristian Ehrhardt#define rte_enic_prefetch(p) rte_prefetch0(p)
468b25d1adSChristian Ehrhardt#else
478b25d1adSChristian Ehrhardt#define rte_enic_prefetch(p) do {} while (0)
488b25d1adSChristian Ehrhardt#endif
498b25d1adSChristian Ehrhardt
508b25d1adSChristian Ehrhardt#ifdef RTE_PMD_PACKET_PREFETCH
518b25d1adSChristian Ehrhardt#define rte_packet_prefetch(p) rte_prefetch1(p)
528b25d1adSChristian Ehrhardt#else
538b25d1adSChristian Ehrhardt#define rte_packet_prefetch(p) do {} while (0)
548b25d1adSChristian Ehrhardt#endif
558b25d1adSChristian Ehrhardt
568b25d1adSChristian Ehrhardtstatic inline uint16_t
578b25d1adSChristian Ehrhardtenic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd)
588b25d1adSChristian Ehrhardt{
598b25d1adSChristian Ehrhardt	return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK;
608b25d1adSChristian Ehrhardt}
618b25d1adSChristian Ehrhardt
628b25d1adSChristian Ehrhardtstatic inline uint16_t
638b25d1adSChristian Ehrhardtenic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd)
648b25d1adSChristian Ehrhardt{
658b25d1adSChristian Ehrhardt	return le16_to_cpu(crd->bytes_written_flags) &
668b25d1adSChristian Ehrhardt			   ~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK;
678b25d1adSChristian Ehrhardt}
688b25d1adSChristian Ehrhardt
698b25d1adSChristian Ehrhardtstatic inline uint8_t
708b25d1adSChristian Ehrhardtenic_cq_rx_desc_packet_error(uint16_t bwflags)
718b25d1adSChristian Ehrhardt{
728b25d1adSChristian Ehrhardt	return (bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) ==
738b25d1adSChristian Ehrhardt		CQ_ENET_RQ_DESC_FLAGS_TRUNCATED;
748b25d1adSChristian Ehrhardt}
758b25d1adSChristian Ehrhardt
768b25d1adSChristian Ehrhardtstatic inline uint8_t
778b25d1adSChristian Ehrhardtenic_cq_rx_desc_eop(uint16_t ciflags)
788b25d1adSChristian Ehrhardt{
798b25d1adSChristian Ehrhardt	return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP)
808b25d1adSChristian Ehrhardt		== CQ_ENET_RQ_DESC_FLAGS_EOP;
818b25d1adSChristian Ehrhardt}
828b25d1adSChristian Ehrhardt
838b25d1adSChristian Ehrhardtstatic inline uint8_t
848b25d1adSChristian Ehrhardtenic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd)
858b25d1adSChristian Ehrhardt{
868b25d1adSChristian Ehrhardt	return (le16_to_cpu(cqrd->q_number_rss_type_flags) &
878b25d1adSChristian Ehrhardt		CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) ==
888b25d1adSChristian Ehrhardt		CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC;
898b25d1adSChristian Ehrhardt}
908b25d1adSChristian Ehrhardt
918b25d1adSChristian Ehrhardtstatic inline uint8_t
928b25d1adSChristian Ehrhardtenic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd)
938b25d1adSChristian Ehrhardt{
948b25d1adSChristian Ehrhardt	return (cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) ==
958b25d1adSChristian Ehrhardt		CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK;
968b25d1adSChristian Ehrhardt}
978b25d1adSChristian Ehrhardt
988b25d1adSChristian Ehrhardtstatic inline uint8_t
998b25d1adSChristian Ehrhardtenic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd)
1008b25d1adSChristian Ehrhardt{
1018b25d1adSChristian Ehrhardt	return (cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) ==
1028b25d1adSChristian Ehrhardt		CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK;
1038b25d1adSChristian Ehrhardt}
1048b25d1adSChristian Ehrhardt
1058b25d1adSChristian Ehrhardtstatic inline uint8_t
1068b25d1adSChristian Ehrhardtenic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd)
1078b25d1adSChristian Ehrhardt{
1088b25d1adSChristian Ehrhardt	return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >>
1098b25d1adSChristian Ehrhardt		CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK);
1108b25d1adSChristian Ehrhardt}
1118b25d1adSChristian Ehrhardt
1128b25d1adSChristian Ehrhardtstatic inline uint32_t
1138b25d1adSChristian Ehrhardtenic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd)
1148b25d1adSChristian Ehrhardt{
1158b25d1adSChristian Ehrhardt	return le32_to_cpu(cqrd->rss_hash);
1168b25d1adSChristian Ehrhardt}
1178b25d1adSChristian Ehrhardt
1188b25d1adSChristian Ehrhardtstatic inline uint16_t
1198b25d1adSChristian Ehrhardtenic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd)
1208b25d1adSChristian Ehrhardt{
1218b25d1adSChristian Ehrhardt	return le16_to_cpu(cqrd->vlan);
1228b25d1adSChristian Ehrhardt}
1238b25d1adSChristian Ehrhardt
1248b25d1adSChristian Ehrhardtstatic inline uint16_t
1258b25d1adSChristian Ehrhardtenic_cq_rx_desc_n_bytes(struct cq_desc *cqd)
1268b25d1adSChristian Ehrhardt{
1278b25d1adSChristian Ehrhardt	struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
1288b25d1adSChristian Ehrhardt	return le16_to_cpu(cqrd->bytes_written_flags) &
1298b25d1adSChristian Ehrhardt		CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK;
1308b25d1adSChristian Ehrhardt}
1318b25d1adSChristian Ehrhardt
1328b25d1adSChristian Ehrhardtstatic inline uint8_t
1338b25d1adSChristian Ehrhardtenic_cq_rx_check_err(struct cq_desc *cqd)
1348b25d1adSChristian Ehrhardt{
1358b25d1adSChristian Ehrhardt	struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
1368b25d1adSChristian Ehrhardt	uint16_t bwflags;
1378b25d1adSChristian Ehrhardt
1388b25d1adSChristian Ehrhardt	bwflags = enic_cq_rx_desc_bwflags(cqrd);
1398b25d1adSChristian Ehrhardt	if (unlikely(enic_cq_rx_desc_packet_error(bwflags)))
1408b25d1adSChristian Ehrhardt		return 1;
1418b25d1adSChristian Ehrhardt	return 0;
1428b25d1adSChristian Ehrhardt}
1438b25d1adSChristian Ehrhardt
1448b25d1adSChristian Ehrhardt/* Lookup table to translate RX CQ flags to mbuf flags. */
1458b25d1adSChristian Ehrhardtstatic inline uint32_t
1468b25d1adSChristian Ehrhardtenic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd)
1478b25d1adSChristian Ehrhardt{
1488b25d1adSChristian Ehrhardt	struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
1498b25d1adSChristian Ehrhardt	uint8_t cqrd_flags = cqrd->flags;
1508b25d1adSChristian Ehrhardt	static const uint32_t cq_type_table[128] __rte_cache_aligned = {
1518b25d1adSChristian Ehrhardt		[0x00] = RTE_PTYPE_UNKNOWN,
1526b3e017eSChristian Ehrhardt		[0x20] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_NONFRAG,
1536b3e017eSChristian Ehrhardt		[0x22] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP,
1546b3e017eSChristian Ehrhardt		[0x24] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_TCP,
1556b3e017eSChristian Ehrhardt		[0x60] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG,
1566b3e017eSChristian Ehrhardt		[0x62] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_UDP,
1576b3e017eSChristian Ehrhardt		[0x64] = RTE_PTYPE_L3_IPV4_EXT_UNKNOWN | RTE_PTYPE_L4_TCP,
1586b3e017eSChristian Ehrhardt		[0x10] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_NONFRAG,
1596b3e017eSChristian Ehrhardt		[0x12] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_UDP,
1606b3e017eSChristian Ehrhardt		[0x14] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_TCP,
1616b3e017eSChristian Ehrhardt		[0x50] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_FRAG,
1626b3e017eSChristian Ehrhardt		[0x52] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_UDP,
1636b3e017eSChristian Ehrhardt		[0x54] = RTE_PTYPE_L3_IPV6_EXT_UNKNOWN | RTE_PTYPE_L4_TCP,
1648b25d1adSChristian Ehrhardt		/* All others reserved */
1658b25d1adSChristian Ehrhardt	};
1668b25d1adSChristian Ehrhardt	cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT
1678b25d1adSChristian Ehrhardt		| CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6
1688b25d1adSChristian Ehrhardt		| CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP;
1698b25d1adSChristian Ehrhardt	return cq_type_table[cqrd_flags];
1708b25d1adSChristian Ehrhardt}
1718b25d1adSChristian Ehrhardt
1728b25d1adSChristian Ehrhardtstatic inline void
1738b25d1adSChristian Ehrhardtenic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf)
1748b25d1adSChristian Ehrhardt{
1758b25d1adSChristian Ehrhardt	struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
1766b3e017eSChristian Ehrhardt	uint16_t ciflags, bwflags, pkt_flags = 0, vlan_tci;
1778b25d1adSChristian Ehrhardt	ciflags = enic_cq_rx_desc_ciflags(cqrd);
1788b25d1adSChristian Ehrhardt	bwflags = enic_cq_rx_desc_bwflags(cqrd);
1796b3e017eSChristian Ehrhardt	vlan_tci = enic_cq_rx_desc_vlan(cqrd);
1808b25d1adSChristian Ehrhardt
1818b25d1adSChristian Ehrhardt	mbuf->ol_flags = 0;
1828b25d1adSChristian Ehrhardt
1838b25d1adSChristian Ehrhardt	/* flags are meaningless if !EOP */
1848b25d1adSChristian Ehrhardt	if (unlikely(!enic_cq_rx_desc_eop(ciflags)))
1858b25d1adSChristian Ehrhardt		goto mbuf_flags_done;
1868b25d1adSChristian Ehrhardt
1876b3e017eSChristian Ehrhardt	/* VLAN STRIPPED flag. The L2 packet type updated here also */
1888b25d1adSChristian Ehrhardt	if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) {
1898b25d1adSChristian Ehrhardt		pkt_flags |= PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED;
1906b3e017eSChristian Ehrhardt		mbuf->packet_type |= RTE_PTYPE_L2_ETHER;
1918b25d1adSChristian Ehrhardt	} else {
192f7a9461eSLuca Boccassi		if (vlan_tci != 0) {
193f7a9461eSLuca Boccassi			pkt_flags |= PKT_RX_VLAN_PKT;
1946b3e017eSChristian Ehrhardt			mbuf->packet_type |= RTE_PTYPE_L2_ETHER_VLAN;
195f7a9461eSLuca Boccassi		} else {
1966b3e017eSChristian Ehrhardt			mbuf->packet_type |= RTE_PTYPE_L2_ETHER;
197f7a9461eSLuca Boccassi		}
1988b25d1adSChristian Ehrhardt	}
1996b3e017eSChristian Ehrhardt	mbuf->vlan_tci = vlan_tci;
2008b25d1adSChristian Ehrhardt
2018b25d1adSChristian Ehrhardt	/* RSS flag */
2028b25d1adSChristian Ehrhardt	if (enic_cq_rx_desc_rss_type(cqrd)) {
2038b25d1adSChristian Ehrhardt		pkt_flags |= PKT_RX_RSS_HASH;
2048b25d1adSChristian Ehrhardt		mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd);
2058b25d1adSChristian Ehrhardt	}
2068b25d1adSChristian Ehrhardt
2078b25d1adSChristian Ehrhardt	/* checksum flags */
2088b25d1adSChristian Ehrhardt	if (!enic_cq_rx_desc_csum_not_calc(cqrd) &&
2098b25d1adSChristian Ehrhardt		(mbuf->packet_type & RTE_PTYPE_L3_IPV4)) {
21032e04ea0SChristian Ehrhardt		uint32_t l4_flags = mbuf->packet_type & RTE_PTYPE_L4_MASK;
21132e04ea0SChristian Ehrhardt
2128b25d1adSChristian Ehrhardt		if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd)))
2138b25d1adSChristian Ehrhardt			pkt_flags |= PKT_RX_IP_CKSUM_BAD;
21432e04ea0SChristian Ehrhardt		if (l4_flags == RTE_PTYPE_L4_UDP ||
21532e04ea0SChristian Ehrhardt		    l4_flags == RTE_PTYPE_L4_TCP) {
2168b25d1adSChristian Ehrhardt			if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd)))
2178b25d1adSChristian Ehrhardt				pkt_flags |= PKT_RX_L4_CKSUM_BAD;
2188b25d1adSChristian Ehrhardt		}
2198b25d1adSChristian Ehrhardt	}
2208b25d1adSChristian Ehrhardt
2218b25d1adSChristian Ehrhardt mbuf_flags_done:
2228b25d1adSChristian Ehrhardt	mbuf->ol_flags = pkt_flags;
2238b25d1adSChristian Ehrhardt}
2248b25d1adSChristian Ehrhardt
2256b3e017eSChristian Ehrhardt/* dummy receive function to replace actual function in
2266b3e017eSChristian Ehrhardt * order to do safe reconfiguration operations.
2276b3e017eSChristian Ehrhardt */
2286b3e017eSChristian Ehrhardtuint16_t
2296b3e017eSChristian Ehrhardtenic_dummy_recv_pkts(__rte_unused void *rx_queue,
2306b3e017eSChristian Ehrhardt		     __rte_unused struct rte_mbuf **rx_pkts,
2316b3e017eSChristian Ehrhardt		     __rte_unused uint16_t nb_pkts)
2326b3e017eSChristian Ehrhardt{
2336b3e017eSChristian Ehrhardt	return 0;
2346b3e017eSChristian Ehrhardt}
2356b3e017eSChristian Ehrhardt
2368b25d1adSChristian Ehrhardtuint16_t
2378b25d1adSChristian Ehrhardtenic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
2388b25d1adSChristian Ehrhardt	       uint16_t nb_pkts)
2398b25d1adSChristian Ehrhardt{
2408b25d1adSChristian Ehrhardt	struct vnic_rq *sop_rq = rx_queue;
2418b25d1adSChristian Ehrhardt	struct vnic_rq *data_rq;
2428b25d1adSChristian Ehrhardt	struct vnic_rq *rq;
2438b25d1adSChristian Ehrhardt	struct enic *enic = vnic_dev_priv(sop_rq->vdev);
2448b25d1adSChristian Ehrhardt	uint16_t cq_idx;
2458b25d1adSChristian Ehrhardt	uint16_t rq_idx;
2468b25d1adSChristian Ehrhardt	uint16_t rq_num;
2478b25d1adSChristian Ehrhardt	struct rte_mbuf *nmb, *rxmb;
2488b25d1adSChristian Ehrhardt	uint16_t nb_rx = 0;
2498b25d1adSChristian Ehrhardt	struct vnic_cq *cq;
2508b25d1adSChristian Ehrhardt	volatile struct cq_desc *cqd_ptr;
2518b25d1adSChristian Ehrhardt	uint8_t color;
2528b25d1adSChristian Ehrhardt	uint16_t seg_length;
2538b25d1adSChristian Ehrhardt	struct rte_mbuf *first_seg = sop_rq->pkt_first_seg;
2548b25d1adSChristian Ehrhardt	struct rte_mbuf *last_seg = sop_rq->pkt_last_seg;
2558b25d1adSChristian Ehrhardt
2568b25d1adSChristian Ehrhardt	cq = &enic->cq[enic_cq_rq(enic, sop_rq->index)];
2578b25d1adSChristian Ehrhardt	cq_idx = cq->to_clean;		/* index of cqd, rqd, mbuf_table */
2588b25d1adSChristian Ehrhardt	cqd_ptr = (struct cq_desc *)(cq->ring.descs) + cq_idx;
2598b25d1adSChristian Ehrhardt
2608b25d1adSChristian Ehrhardt	data_rq = &enic->rq[sop_rq->data_queue_idx];
2618b25d1adSChristian Ehrhardt
2628b25d1adSChristian Ehrhardt	while (nb_rx < nb_pkts) {
2638b25d1adSChristian Ehrhardt		volatile struct rq_enet_desc *rqd_ptr;
2648b25d1adSChristian Ehrhardt		dma_addr_t dma_addr;
2658b25d1adSChristian Ehrhardt		struct cq_desc cqd;
2668b25d1adSChristian Ehrhardt		uint8_t packet_error;
2678b25d1adSChristian Ehrhardt		uint16_t ciflags;
2688b25d1adSChristian Ehrhardt
2698b25d1adSChristian Ehrhardt		/* Check for pkts available */
2708b25d1adSChristian Ehrhardt		color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT)
2718b25d1adSChristian Ehrhardt			& CQ_DESC_COLOR_MASK;
2728b25d1adSChristian Ehrhardt		if (color == cq->last_color)
2738b25d1adSChristian Ehrhardt			break;
2748b25d1adSChristian Ehrhardt
2758b25d1adSChristian Ehrhardt		/* Get the cq descriptor and extract rq info from it */
2768b25d1adSChristian Ehrhardt		cqd = *cqd_ptr;
2778b25d1adSChristian Ehrhardt		rq_num = cqd.q_number & CQ_DESC_Q_NUM_MASK;
2788b25d1adSChristian Ehrhardt		rq_idx = cqd.completed_index & CQ_DESC_COMP_NDX_MASK;
2798b25d1adSChristian Ehrhardt
2808b25d1adSChristian Ehrhardt		rq = &enic->rq[rq_num];
2818b25d1adSChristian Ehrhardt		rqd_ptr = ((struct rq_enet_desc *)rq->ring.descs) + rq_idx;
2828b25d1adSChristian Ehrhardt
2838b25d1adSChristian Ehrhardt		/* allocate a new mbuf */
2848b25d1adSChristian Ehrhardt		nmb = rte_mbuf_raw_alloc(rq->mp);
2858b25d1adSChristian Ehrhardt		if (nmb == NULL) {
2868b25d1adSChristian Ehrhardt			rte_atomic64_inc(&enic->soft_stats.rx_nombuf);
2878b25d1adSChristian Ehrhardt			break;
2888b25d1adSChristian Ehrhardt		}
2898b25d1adSChristian Ehrhardt
2908b25d1adSChristian Ehrhardt		/* A packet error means descriptor and data are untrusted */
2918b25d1adSChristian Ehrhardt		packet_error = enic_cq_rx_check_err(&cqd);
2928b25d1adSChristian Ehrhardt
2938b25d1adSChristian Ehrhardt		/* Get the mbuf to return and replace with one just allocated */
2948b25d1adSChristian Ehrhardt		rxmb = rq->mbuf_ring[rq_idx];
2958b25d1adSChristian Ehrhardt		rq->mbuf_ring[rq_idx] = nmb;
2968b25d1adSChristian Ehrhardt
2978b25d1adSChristian Ehrhardt		/* Increment cqd, rqd, mbuf_table index */
2988b25d1adSChristian Ehrhardt		cq_idx++;
2998b25d1adSChristian Ehrhardt		if (unlikely(cq_idx == cq->ring.desc_count)) {
3008b25d1adSChristian Ehrhardt			cq_idx = 0;
3018b25d1adSChristian Ehrhardt			cq->last_color = cq->last_color ? 0 : 1;
3028b25d1adSChristian Ehrhardt		}
3038b25d1adSChristian Ehrhardt
3048b25d1adSChristian Ehrhardt		/* Prefetch next mbuf & desc while processing current one */
3058b25d1adSChristian Ehrhardt		cqd_ptr = (struct cq_desc *)(cq->ring.descs) + cq_idx;
3068b25d1adSChristian Ehrhardt		rte_enic_prefetch(cqd_ptr);
3078b25d1adSChristian Ehrhardt
3088b25d1adSChristian Ehrhardt		ciflags = enic_cq_rx_desc_ciflags(
3098b25d1adSChristian Ehrhardt			(struct cq_enet_rq_desc *)&cqd);
3108b25d1adSChristian Ehrhardt
3118b25d1adSChristian Ehrhardt		/* Push descriptor for newly allocated mbuf */
3127b53c036SRicardo Salveti		nmb->data_off = RTE_PKTMBUF_HEADROOM;
3138b25d1adSChristian Ehrhardt		dma_addr = (dma_addr_t)(nmb->buf_physaddr +
3148b25d1adSChristian Ehrhardt					RTE_PKTMBUF_HEADROOM);
3158b25d1adSChristian Ehrhardt		rq_enet_desc_enc(rqd_ptr, dma_addr,
3168b25d1adSChristian Ehrhardt				(rq->is_sop ? RQ_ENET_TYPE_ONLY_SOP
3178b25d1adSChristian Ehrhardt				: RQ_ENET_TYPE_NOT_SOP),
3188b25d1adSChristian Ehrhardt				nmb->buf_len - RTE_PKTMBUF_HEADROOM);
3198b25d1adSChristian Ehrhardt
3208b25d1adSChristian Ehrhardt		/* Fill in the rest of the mbuf */
3218b25d1adSChristian Ehrhardt		seg_length = enic_cq_rx_desc_n_bytes(&cqd);
3228b25d1adSChristian Ehrhardt
3238b25d1adSChristian Ehrhardt		if (rq->is_sop) {
3248b25d1adSChristian Ehrhardt			first_seg = rxmb;
3258b25d1adSChristian Ehrhardt			first_seg->nb_segs = 1;
3268b25d1adSChristian Ehrhardt			first_seg->pkt_len = seg_length;
3278b25d1adSChristian Ehrhardt		} else {
3288b25d1adSChristian Ehrhardt			first_seg->pkt_len = (uint16_t)(first_seg->pkt_len
3298b25d1adSChristian Ehrhardt							+ seg_length);
3308b25d1adSChristian Ehrhardt			first_seg->nb_segs++;
3318b25d1adSChristian Ehrhardt			last_seg->next = rxmb;
3328b25d1adSChristian Ehrhardt		}
3338b25d1adSChristian Ehrhardt
3348b25d1adSChristian Ehrhardt		rxmb->next = NULL;
3358b25d1adSChristian Ehrhardt		rxmb->port = enic->port_id;
3368b25d1adSChristian Ehrhardt		rxmb->data_len = seg_length;
3378b25d1adSChristian Ehrhardt
3388b25d1adSChristian Ehrhardt		rq->rx_nb_hold++;
3398b25d1adSChristian Ehrhardt
3408b25d1adSChristian Ehrhardt		if (!(enic_cq_rx_desc_eop(ciflags))) {
3418b25d1adSChristian Ehrhardt			last_seg = rxmb;
3428b25d1adSChristian Ehrhardt			continue;
3438b25d1adSChristian Ehrhardt		}
3448b25d1adSChristian Ehrhardt
3458b25d1adSChristian Ehrhardt		/* cq rx flags are only valid if eop bit is set */
3468b25d1adSChristian Ehrhardt		first_seg->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd);
3478b25d1adSChristian Ehrhardt		enic_cq_rx_to_pkt_flags(&cqd, first_seg);
3488b25d1adSChristian Ehrhardt
3498b25d1adSChristian Ehrhardt		if (unlikely(packet_error)) {
3508b25d1adSChristian Ehrhardt			rte_pktmbuf_free(first_seg);
3518b25d1adSChristian Ehrhardt			rte_atomic64_inc(&enic->soft_stats.rx_packet_errors);
3528b25d1adSChristian Ehrhardt			continue;
3538b25d1adSChristian Ehrhardt		}
3548b25d1adSChristian Ehrhardt
3558b25d1adSChristian Ehrhardt
3568b25d1adSChristian Ehrhardt		/* prefetch mbuf data for caller */
3578b25d1adSChristian Ehrhardt		rte_packet_prefetch(RTE_PTR_ADD(first_seg->buf_addr,
3588b25d1adSChristian Ehrhardt				    RTE_PKTMBUF_HEADROOM));
3598b25d1adSChristian Ehrhardt
3608b25d1adSChristian Ehrhardt		/* store the mbuf address into the next entry of the array */
3618b25d1adSChristian Ehrhardt		rx_pkts[nb_rx++] = first_seg;
3628b25d1adSChristian Ehrhardt	}
3638b25d1adSChristian Ehrhardt
3648b25d1adSChristian Ehrhardt	sop_rq->pkt_first_seg = first_seg;
3658b25d1adSChristian Ehrhardt	sop_rq->pkt_last_seg = last_seg;
3668b25d1adSChristian Ehrhardt
3678b25d1adSChristian Ehrhardt	cq->to_clean = cq_idx;
3688b25d1adSChristian Ehrhardt
3698b25d1adSChristian Ehrhardt	if ((sop_rq->rx_nb_hold + data_rq->rx_nb_hold) >
3708b25d1adSChristian Ehrhardt	    sop_rq->rx_free_thresh) {
3718b25d1adSChristian Ehrhardt		if (data_rq->in_use) {
3728b25d1adSChristian Ehrhardt			data_rq->posted_index =
3738b25d1adSChristian Ehrhardt				enic_ring_add(data_rq->ring.desc_count,
3748b25d1adSChristian Ehrhardt					      data_rq->posted_index,
3758b25d1adSChristian Ehrhardt					      data_rq->rx_nb_hold);
3768b25d1adSChristian Ehrhardt			data_rq->rx_nb_hold = 0;
3778b25d1adSChristian Ehrhardt		}
3788b25d1adSChristian Ehrhardt		sop_rq->posted_index = enic_ring_add(sop_rq->ring.desc_count,
3798b25d1adSChristian Ehrhardt						     sop_rq->posted_index,
3808b25d1adSChristian Ehrhardt						     sop_rq->rx_nb_hold);
3818b25d1adSChristian Ehrhardt		sop_rq->rx_nb_hold = 0;
3828b25d1adSChristian Ehrhardt
3838b25d1adSChristian Ehrhardt		rte_mb();
3848b25d1adSChristian Ehrhardt		if (data_rq->in_use)
3858b25d1adSChristian Ehrhardt			iowrite32(data_rq->posted_index,
3868b25d1adSChristian Ehrhardt				  &data_rq->ctrl->posted_index);
3878b25d1adSChristian Ehrhardt		rte_compiler_barrier();
3888b25d1adSChristian Ehrhardt		iowrite32(sop_rq->posted_index, &sop_rq->ctrl->posted_index);
3898b25d1adSChristian Ehrhardt	}
3908b25d1adSChristian Ehrhardt
3918b25d1adSChristian Ehrhardt
3928b25d1adSChristian Ehrhardt	return nb_rx;
3938b25d1adSChristian Ehrhardt}
3948b25d1adSChristian Ehrhardt
3958b25d1adSChristian Ehrhardtstatic inline void enic_free_wq_bufs(struct vnic_wq *wq, u16 completed_index)
3968b25d1adSChristian Ehrhardt{
3978b25d1adSChristian Ehrhardt	struct vnic_wq_buf *buf;
3988b25d1adSChristian Ehrhardt	struct rte_mbuf *m, *free[ENIC_MAX_WQ_DESCS];
3998b25d1adSChristian Ehrhardt	unsigned int nb_to_free, nb_free = 0, i;
4008b25d1adSChristian Ehrhardt	struct rte_mempool *pool;
4018b25d1adSChristian Ehrhardt	unsigned int tail_idx;
4028b25d1adSChristian Ehrhardt	unsigned int desc_count = wq->ring.desc_count;
4038b25d1adSChristian Ehrhardt
4048b25d1adSChristian Ehrhardt	nb_to_free = enic_ring_sub(desc_count, wq->tail_idx, completed_index)
4058b25d1adSChristian Ehrhardt				   + 1;
4068b25d1adSChristian Ehrhardt	tail_idx = wq->tail_idx;
4078b25d1adSChristian Ehrhardt	buf = &wq->bufs[tail_idx];
4088b25d1adSChristian Ehrhardt	pool = ((struct rte_mbuf *)buf->mb)->pool;
4098b25d1adSChristian Ehrhardt	for (i = 0; i < nb_to_free; i++) {
4108b25d1adSChristian Ehrhardt		buf = &wq->bufs[tail_idx];
4115d4e5dcdSRicardo Salveti		m = __rte_pktmbuf_prefree_seg((struct rte_mbuf *)(buf->mb));
4125d4e5dcdSRicardo Salveti		buf->mb = NULL;
4135d4e5dcdSRicardo Salveti
4145d4e5dcdSRicardo Salveti		if (unlikely(m == NULL)) {
4155d4e5dcdSRicardo Salveti			tail_idx = enic_ring_incr(desc_count, tail_idx);
4165d4e5dcdSRicardo Salveti			continue;
4175d4e5dcdSRicardo Salveti		}
4185d4e5dcdSRicardo Salveti
4198b25d1adSChristian Ehrhardt		if (likely(m->pool == pool)) {
420a41e6ff1SRicardo Salveti			RTE_ASSERT(nb_free < ENIC_MAX_WQ_DESCS);
4218b25d1adSChristian Ehrhardt			free[nb_free++] = m;
4228b25d1adSChristian Ehrhardt		} else {
4238b25d1adSChristian Ehrhardt			rte_mempool_put_bulk(pool, (void *)free, nb_free);
4248b25d1adSChristian Ehrhardt			free[0] = m;
4258b25d1adSChristian Ehrhardt			nb_free = 1;
4268b25d1adSChristian Ehrhardt			pool = m->pool;
4278b25d1adSChristian Ehrhardt		}
4288b25d1adSChristian Ehrhardt		tail_idx = enic_ring_incr(desc_count, tail_idx);
4298b25d1adSChristian Ehrhardt	}
4308b25d1adSChristian Ehrhardt
431fdd2322bSLuca Boccassi	if (nb_free > 0)
432fdd2322bSLuca Boccassi		rte_mempool_put_bulk(pool, (void **)free, nb_free);
4338b25d1adSChristian Ehrhardt
4348b25d1adSChristian Ehrhardt	wq->tail_idx = tail_idx;
4358b25d1adSChristian Ehrhardt	wq->ring.desc_avail += nb_to_free;
4368b25d1adSChristian Ehrhardt}
4378b25d1adSChristian Ehrhardt
4388b25d1adSChristian Ehrhardtunsigned int enic_cleanup_wq(__rte_unused struct enic *enic, struct vnic_wq *wq)
4398b25d1adSChristian Ehrhardt{
4408b25d1adSChristian Ehrhardt	u16 completed_index;
4418b25d1adSChristian Ehrhardt
4428b25d1adSChristian Ehrhardt	completed_index = *((uint32_t *)wq->cqmsg_rz->addr) & 0xffff;
4438b25d1adSChristian Ehrhardt
4448b25d1adSChristian Ehrhardt	if (wq->last_completed_index != completed_index) {
4458b25d1adSChristian Ehrhardt		enic_free_wq_bufs(wq, completed_index);
4468b25d1adSChristian Ehrhardt		wq->last_completed_index = completed_index;
4478b25d1adSChristian Ehrhardt	}
4488b25d1adSChristian Ehrhardt	return 0;
4498b25d1adSChristian Ehrhardt}
4508b25d1adSChristian Ehrhardt
4518b25d1adSChristian Ehrhardtuint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
4528b25d1adSChristian Ehrhardt	uint16_t nb_pkts)
4538b25d1adSChristian Ehrhardt{
4548b25d1adSChristian Ehrhardt	uint16_t index;
4558b25d1adSChristian Ehrhardt	unsigned int pkt_len, data_len;
4568b25d1adSChristian Ehrhardt	unsigned int nb_segs;
4578b25d1adSChristian Ehrhardt	struct rte_mbuf *tx_pkt;
4588b25d1adSChristian Ehrhardt	struct vnic_wq *wq = (struct vnic_wq *)tx_queue;
4598b25d1adSChristian Ehrhardt	struct enic *enic = vnic_dev_priv(wq->vdev);
4608b25d1adSChristian Ehrhardt	unsigned short vlan_id;
4618b25d1adSChristian Ehrhardt	uint64_t ol_flags;
4628b25d1adSChristian Ehrhardt	uint64_t ol_flags_mask;
4638b25d1adSChristian Ehrhardt	unsigned int wq_desc_avail;
4648b25d1adSChristian Ehrhardt	int head_idx;
4658b25d1adSChristian Ehrhardt	struct vnic_wq_buf *buf;
4668b25d1adSChristian Ehrhardt	unsigned int desc_count;
4678b25d1adSChristian Ehrhardt	struct wq_enet_desc *descs, *desc_p, desc_tmp;
4688b25d1adSChristian Ehrhardt	uint16_t mss;
4698b25d1adSChristian Ehrhardt	uint8_t vlan_tag_insert;
4708b25d1adSChristian Ehrhardt	uint8_t eop;
4718b25d1adSChristian Ehrhardt	uint64_t bus_addr;
4728b25d1adSChristian Ehrhardt
4738b25d1adSChristian Ehrhardt	enic_cleanup_wq(enic, wq);
4748b25d1adSChristian Ehrhardt	wq_desc_avail = vnic_wq_desc_avail(wq);
4758b25d1adSChristian Ehrhardt	head_idx = wq->head_idx;
4768b25d1adSChristian Ehrhardt	desc_count = wq->ring.desc_count;
4778b25d1adSChristian Ehrhardt	ol_flags_mask = PKT_TX_VLAN_PKT | PKT_TX_IP_CKSUM | PKT_TX_L4_MASK;
4788b25d1adSChristian Ehrhardt
4798b25d1adSChristian Ehrhardt	nb_pkts = RTE_MIN(nb_pkts, ENIC_TX_XMIT_MAX);
4808b25d1adSChristian Ehrhardt
4818b25d1adSChristian Ehrhardt	for (index = 0; index < nb_pkts; index++) {
4828b25d1adSChristian Ehrhardt		tx_pkt = *tx_pkts++;
483ce3d555eSChristian Ehrhardt		pkt_len = tx_pkt->pkt_len;
484ce3d555eSChristian Ehrhardt		data_len = tx_pkt->data_len;
485ce3d555eSChristian Ehrhardt		ol_flags = tx_pkt->ol_flags;
4868b25d1adSChristian Ehrhardt		nb_segs = tx_pkt->nb_segs;
487ce3d555eSChristian Ehrhardt
488ce3d555eSChristian Ehrhardt		if (pkt_len > ENIC_TX_MAX_PKT_SIZE) {
489ce3d555eSChristian Ehrhardt			rte_pktmbuf_free(tx_pkt);
490ce3d555eSChristian Ehrhardt			rte_atomic64_inc(&enic->soft_stats.tx_oversized);
491ce3d555eSChristian Ehrhardt			continue;
492ce3d555eSChristian Ehrhardt		}
493ce3d555eSChristian Ehrhardt
4948b25d1adSChristian Ehrhardt		if (nb_segs > wq_desc_avail) {
4958b25d1adSChristian Ehrhardt			if (index > 0)
4968b25d1adSChristian Ehrhardt				goto post;
4978b25d1adSChristian Ehrhardt			goto done;
4988b25d1adSChristian Ehrhardt		}
4998b25d1adSChristian Ehrhardt
5008b25d1adSChristian Ehrhardt		mss = 0;
5018b25d1adSChristian Ehrhardt		vlan_id = 0;
5028b25d1adSChristian Ehrhardt		vlan_tag_insert = 0;
5038b25d1adSChristian Ehrhardt		bus_addr = (dma_addr_t)
5048b25d1adSChristian Ehrhardt			   (tx_pkt->buf_physaddr + tx_pkt->data_off);
5058b25d1adSChristian Ehrhardt
5068b25d1adSChristian Ehrhardt		descs = (struct wq_enet_desc *)wq->ring.descs;
5078b25d1adSChristian Ehrhardt		desc_p = descs + head_idx;
5088b25d1adSChristian Ehrhardt
5098b25d1adSChristian Ehrhardt		eop = (data_len == pkt_len);
5108b25d1adSChristian Ehrhardt
5118b25d1adSChristian Ehrhardt		if (ol_flags & ol_flags_mask) {
5128b25d1adSChristian Ehrhardt			if (ol_flags & PKT_TX_VLAN_PKT) {
5138b25d1adSChristian Ehrhardt				vlan_tag_insert = 1;
5148b25d1adSChristian Ehrhardt				vlan_id = tx_pkt->vlan_tci;
5158b25d1adSChristian Ehrhardt			}
5168b25d1adSChristian Ehrhardt
5178b25d1adSChristian Ehrhardt			if (ol_flags & PKT_TX_IP_CKSUM)
5188b25d1adSChristian Ehrhardt				mss |= ENIC_CALC_IP_CKSUM;
5198b25d1adSChristian Ehrhardt
5208b25d1adSChristian Ehrhardt			/* Nic uses just 1 bit for UDP and TCP */
5218b25d1adSChristian Ehrhardt			switch (ol_flags & PKT_TX_L4_MASK) {
5228b25d1adSChristian Ehrhardt			case PKT_TX_TCP_CKSUM:
5238b25d1adSChristian Ehrhardt			case PKT_TX_UDP_CKSUM:
5248b25d1adSChristian Ehrhardt				mss |= ENIC_CALC_TCP_UDP_CKSUM;
5258b25d1adSChristian Ehrhardt				break;
5268b25d1adSChristian Ehrhardt			}
5278b25d1adSChristian Ehrhardt		}
5288b25d1adSChristian Ehrhardt
5298b25d1adSChristian Ehrhardt		wq_enet_desc_enc(&desc_tmp, bus_addr, data_len, mss, 0, 0, eop,
5308b25d1adSChristian Ehrhardt				 eop, 0, vlan_tag_insert, vlan_id, 0);
5318b25d1adSChristian Ehrhardt
5328b25d1adSChristian Ehrhardt		*desc_p = desc_tmp;
5338b25d1adSChristian Ehrhardt		buf = &wq->bufs[head_idx];
5348b25d1adSChristian Ehrhardt		buf->mb = (void *)tx_pkt;
5358b25d1adSChristian Ehrhardt		head_idx = enic_ring_incr(desc_count, head_idx);
5368b25d1adSChristian Ehrhardt		wq_desc_avail--;
5378b25d1adSChristian Ehrhardt
5388b25d1adSChristian Ehrhardt		if (!eop) {
5398b25d1adSChristian Ehrhardt			for (tx_pkt = tx_pkt->next; tx_pkt; tx_pkt =
5408b25d1adSChristian Ehrhardt			    tx_pkt->next) {
5418b25d1adSChristian Ehrhardt				data_len = tx_pkt->data_len;
5428b25d1adSChristian Ehrhardt
5438b25d1adSChristian Ehrhardt				if (tx_pkt->next == NULL)
5448b25d1adSChristian Ehrhardt					eop = 1;
5458b25d1adSChristian Ehrhardt				desc_p = descs + head_idx;
5468b25d1adSChristian Ehrhardt				bus_addr = (dma_addr_t)(tx_pkt->buf_physaddr
5478b25d1adSChristian Ehrhardt					   + tx_pkt->data_off);
5488b25d1adSChristian Ehrhardt				wq_enet_desc_enc((struct wq_enet_desc *)
5498b25d1adSChristian Ehrhardt						 &desc_tmp, bus_addr, data_len,
5508b25d1adSChristian Ehrhardt						 mss, 0, 0, eop, eop, 0,
5518b25d1adSChristian Ehrhardt						 vlan_tag_insert, vlan_id, 0);
5528b25d1adSChristian Ehrhardt
5538b25d1adSChristian Ehrhardt				*desc_p = desc_tmp;
5548b25d1adSChristian Ehrhardt				buf = &wq->bufs[head_idx];
5558b25d1adSChristian Ehrhardt				buf->mb = (void *)tx_pkt;
5568b25d1adSChristian Ehrhardt				head_idx = enic_ring_incr(desc_count, head_idx);
5578b25d1adSChristian Ehrhardt				wq_desc_avail--;
5588b25d1adSChristian Ehrhardt			}
5598b25d1adSChristian Ehrhardt		}
5608b25d1adSChristian Ehrhardt	}
5618b25d1adSChristian Ehrhardt post:
5628b25d1adSChristian Ehrhardt	rte_wmb();
5638b25d1adSChristian Ehrhardt	iowrite32(head_idx, &wq->ctrl->posted_index);
5648b25d1adSChristian Ehrhardt done:
5658b25d1adSChristian Ehrhardt	wq->ring.desc_avail = wq_desc_avail;
5668b25d1adSChristian Ehrhardt	wq->head_idx = head_idx;
5678b25d1adSChristian Ehrhardt
5688b25d1adSChristian Ehrhardt	return index;
5698b25d1adSChristian Ehrhardt}
5708b25d1adSChristian Ehrhardt
5718b25d1adSChristian Ehrhardt
572