virtio_ethdev.c revision 93f15e30
1/*-
2 *   BSD LICENSE
3 *
4 *   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
5 *   All rights reserved.
6 *
7 *   Redistribution and use in source and binary forms, with or without
8 *   modification, are permitted provided that the following conditions
9 *   are met:
10 *
11 *     * Redistributions of source code must retain the above copyright
12 *       notice, this list of conditions and the following disclaimer.
13 *     * Redistributions in binary form must reproduce the above copyright
14 *       notice, this list of conditions and the following disclaimer in
15 *       the documentation and/or other materials provided with the
16 *       distribution.
17 *     * Neither the name of Intel Corporation nor the names of its
18 *       contributors may be used to endorse or promote products derived
19 *       from this software without specific prior written permission.
20 *
21 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34#include <stdint.h>
35#include <string.h>
36#include <stdio.h>
37#include <errno.h>
38#include <unistd.h>
39
40#include <rte_ethdev.h>
41#include <rte_memcpy.h>
42#include <rte_string_fns.h>
43#include <rte_memzone.h>
44#include <rte_malloc.h>
45#include <rte_atomic.h>
46#include <rte_branch_prediction.h>
47#include <rte_pci.h>
48#include <rte_ether.h>
49#include <rte_common.h>
50#include <rte_errno.h>
51
52#include <rte_memory.h>
53#include <rte_eal.h>
54#include <rte_dev.h>
55
56#include "virtio_ethdev.h"
57#include "virtio_pci.h"
58#include "virtio_logs.h"
59#include "virtqueue.h"
60#include "virtio_rxtx.h"
61
62static int eth_virtio_dev_uninit(struct rte_eth_dev *eth_dev);
63static int  virtio_dev_configure(struct rte_eth_dev *dev);
64static int  virtio_dev_start(struct rte_eth_dev *dev);
65static void virtio_dev_stop(struct rte_eth_dev *dev);
66static void virtio_dev_promiscuous_enable(struct rte_eth_dev *dev);
67static void virtio_dev_promiscuous_disable(struct rte_eth_dev *dev);
68static void virtio_dev_allmulticast_enable(struct rte_eth_dev *dev);
69static void virtio_dev_allmulticast_disable(struct rte_eth_dev *dev);
70static void virtio_dev_info_get(struct rte_eth_dev *dev,
71				struct rte_eth_dev_info *dev_info);
72static int virtio_dev_link_update(struct rte_eth_dev *dev,
73	__rte_unused int wait_to_complete);
74
75static void virtio_set_hwaddr(struct virtio_hw *hw);
76static void virtio_get_hwaddr(struct virtio_hw *hw);
77
78static void virtio_dev_stats_get(struct rte_eth_dev *dev,
79				 struct rte_eth_stats *stats);
80static int virtio_dev_xstats_get(struct rte_eth_dev *dev,
81				 struct rte_eth_xstat *xstats, unsigned n);
82static int virtio_dev_xstats_get_names(struct rte_eth_dev *dev,
83				       struct rte_eth_xstat_name *xstats_names,
84				       unsigned limit);
85static void virtio_dev_stats_reset(struct rte_eth_dev *dev);
86static void virtio_dev_free_mbufs(struct rte_eth_dev *dev);
87static int virtio_vlan_filter_set(struct rte_eth_dev *dev,
88				uint16_t vlan_id, int on);
89static void virtio_mac_addr_add(struct rte_eth_dev *dev,
90				struct ether_addr *mac_addr,
91				uint32_t index, uint32_t vmdq __rte_unused);
92static void virtio_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index);
93static void virtio_mac_addr_set(struct rte_eth_dev *dev,
94				struct ether_addr *mac_addr);
95
96static int virtio_dev_queue_stats_mapping_set(
97	__rte_unused struct rte_eth_dev *eth_dev,
98	__rte_unused uint16_t queue_id,
99	__rte_unused uint8_t stat_idx,
100	__rte_unused uint8_t is_rx);
101
102/*
103 * The set of PCI devices this driver supports
104 */
105static const struct rte_pci_id pci_id_virtio_map[] = {
106	{ RTE_PCI_DEVICE(VIRTIO_PCI_VENDORID, VIRTIO_PCI_LEGACY_DEVICEID_NET) },
107	{ RTE_PCI_DEVICE(VIRTIO_PCI_VENDORID, VIRTIO_PCI_MODERN_DEVICEID_NET) },
108	{ .vendor_id = 0, /* sentinel */ },
109};
110
111struct rte_virtio_xstats_name_off {
112	char name[RTE_ETH_XSTATS_NAME_SIZE];
113	unsigned offset;
114};
115
116/* [rt]x_qX_ is prepended to the name string here */
117static const struct rte_virtio_xstats_name_off rte_virtio_rxq_stat_strings[] = {
118	{"good_packets",           offsetof(struct virtnet_rx, stats.packets)},
119	{"good_bytes",             offsetof(struct virtnet_rx, stats.bytes)},
120	{"errors",                 offsetof(struct virtnet_rx, stats.errors)},
121	{"multicast_packets",      offsetof(struct virtnet_rx, stats.multicast)},
122	{"broadcast_packets",      offsetof(struct virtnet_rx, stats.broadcast)},
123	{"undersize_packets",      offsetof(struct virtnet_rx, stats.size_bins[0])},
124	{"size_64_packets",        offsetof(struct virtnet_rx, stats.size_bins[1])},
125	{"size_65_127_packets",    offsetof(struct virtnet_rx, stats.size_bins[2])},
126	{"size_128_255_packets",   offsetof(struct virtnet_rx, stats.size_bins[3])},
127	{"size_256_511_packets",   offsetof(struct virtnet_rx, stats.size_bins[4])},
128	{"size_512_1023_packets",  offsetof(struct virtnet_rx, stats.size_bins[5])},
129	{"size_1024_1518_packets", offsetof(struct virtnet_rx, stats.size_bins[6])},
130	{"size_1519_max_packets",  offsetof(struct virtnet_rx, stats.size_bins[7])},
131};
132
133/* [rt]x_qX_ is prepended to the name string here */
134static const struct rte_virtio_xstats_name_off rte_virtio_txq_stat_strings[] = {
135	{"good_packets",           offsetof(struct virtnet_tx, stats.packets)},
136	{"good_bytes",             offsetof(struct virtnet_tx, stats.bytes)},
137	{"errors",                 offsetof(struct virtnet_tx, stats.errors)},
138	{"multicast_packets",      offsetof(struct virtnet_tx, stats.multicast)},
139	{"broadcast_packets",      offsetof(struct virtnet_tx, stats.broadcast)},
140	{"undersize_packets",      offsetof(struct virtnet_tx, stats.size_bins[0])},
141	{"size_64_packets",        offsetof(struct virtnet_tx, stats.size_bins[1])},
142	{"size_65_127_packets",    offsetof(struct virtnet_tx, stats.size_bins[2])},
143	{"size_128_255_packets",   offsetof(struct virtnet_tx, stats.size_bins[3])},
144	{"size_256_511_packets",   offsetof(struct virtnet_tx, stats.size_bins[4])},
145	{"size_512_1023_packets",  offsetof(struct virtnet_tx, stats.size_bins[5])},
146	{"size_1024_1518_packets", offsetof(struct virtnet_tx, stats.size_bins[6])},
147	{"size_1519_max_packets",  offsetof(struct virtnet_tx, stats.size_bins[7])},
148};
149
150#define VIRTIO_NB_RXQ_XSTATS (sizeof(rte_virtio_rxq_stat_strings) / \
151			    sizeof(rte_virtio_rxq_stat_strings[0]))
152#define VIRTIO_NB_TXQ_XSTATS (sizeof(rte_virtio_txq_stat_strings) / \
153			    sizeof(rte_virtio_txq_stat_strings[0]))
154
155struct virtio_hw_internal virtio_hw_internal[RTE_MAX_ETHPORTS];
156
157static int
158virtio_send_command(struct virtnet_ctl *cvq, struct virtio_pmd_ctrl *ctrl,
159		int *dlen, int pkt_num)
160{
161	uint32_t head, i;
162	int k, sum = 0;
163	virtio_net_ctrl_ack status = ~0;
164	struct virtio_pmd_ctrl result;
165	struct virtqueue *vq;
166
167	ctrl->status = status;
168
169	if (!cvq || !cvq->vq) {
170		PMD_INIT_LOG(ERR, "Control queue is not supported.");
171		return -1;
172	}
173	vq = cvq->vq;
174	head = vq->vq_desc_head_idx;
175
176	PMD_INIT_LOG(DEBUG, "vq->vq_desc_head_idx = %d, status = %d, "
177		"vq->hw->cvq = %p vq = %p",
178		vq->vq_desc_head_idx, status, vq->hw->cvq, vq);
179
180	if ((vq->vq_free_cnt < ((uint32_t)pkt_num + 2)) || (pkt_num < 1))
181		return -1;
182
183	memcpy(cvq->virtio_net_hdr_mz->addr, ctrl,
184		sizeof(struct virtio_pmd_ctrl));
185
186	/*
187	 * Format is enforced in qemu code:
188	 * One TX packet for header;
189	 * At least one TX packet per argument;
190	 * One RX packet for ACK.
191	 */
192	vq->vq_ring.desc[head].flags = VRING_DESC_F_NEXT;
193	vq->vq_ring.desc[head].addr = cvq->virtio_net_hdr_mem;
194	vq->vq_ring.desc[head].len = sizeof(struct virtio_net_ctrl_hdr);
195	vq->vq_free_cnt--;
196	i = vq->vq_ring.desc[head].next;
197
198	for (k = 0; k < pkt_num; k++) {
199		vq->vq_ring.desc[i].flags = VRING_DESC_F_NEXT;
200		vq->vq_ring.desc[i].addr = cvq->virtio_net_hdr_mem
201			+ sizeof(struct virtio_net_ctrl_hdr)
202			+ sizeof(ctrl->status) + sizeof(uint8_t)*sum;
203		vq->vq_ring.desc[i].len = dlen[k];
204		sum += dlen[k];
205		vq->vq_free_cnt--;
206		i = vq->vq_ring.desc[i].next;
207	}
208
209	vq->vq_ring.desc[i].flags = VRING_DESC_F_WRITE;
210	vq->vq_ring.desc[i].addr = cvq->virtio_net_hdr_mem
211			+ sizeof(struct virtio_net_ctrl_hdr);
212	vq->vq_ring.desc[i].len = sizeof(ctrl->status);
213	vq->vq_free_cnt--;
214
215	vq->vq_desc_head_idx = vq->vq_ring.desc[i].next;
216
217	vq_update_avail_ring(vq, head);
218	vq_update_avail_idx(vq);
219
220	PMD_INIT_LOG(DEBUG, "vq->vq_queue_index = %d", vq->vq_queue_index);
221
222	virtqueue_notify(vq);
223
224	rte_rmb();
225	while (VIRTQUEUE_NUSED(vq) == 0) {
226		rte_rmb();
227		usleep(100);
228	}
229
230	while (VIRTQUEUE_NUSED(vq)) {
231		uint32_t idx, desc_idx, used_idx;
232		struct vring_used_elem *uep;
233
234		used_idx = (uint32_t)(vq->vq_used_cons_idx
235				& (vq->vq_nentries - 1));
236		uep = &vq->vq_ring.used->ring[used_idx];
237		idx = (uint32_t) uep->id;
238		desc_idx = idx;
239
240		while (vq->vq_ring.desc[desc_idx].flags & VRING_DESC_F_NEXT) {
241			desc_idx = vq->vq_ring.desc[desc_idx].next;
242			vq->vq_free_cnt++;
243		}
244
245		vq->vq_ring.desc[desc_idx].next = vq->vq_desc_head_idx;
246		vq->vq_desc_head_idx = idx;
247
248		vq->vq_used_cons_idx++;
249		vq->vq_free_cnt++;
250	}
251
252	PMD_INIT_LOG(DEBUG, "vq->vq_free_cnt=%d\nvq->vq_desc_head_idx=%d",
253			vq->vq_free_cnt, vq->vq_desc_head_idx);
254
255	memcpy(&result, cvq->virtio_net_hdr_mz->addr,
256			sizeof(struct virtio_pmd_ctrl));
257
258	return result.status;
259}
260
261static int
262virtio_set_multiple_queues(struct rte_eth_dev *dev, uint16_t nb_queues)
263{
264	struct virtio_hw *hw = dev->data->dev_private;
265	struct virtio_pmd_ctrl ctrl;
266	int dlen[1];
267	int ret;
268
269	ctrl.hdr.class = VIRTIO_NET_CTRL_MQ;
270	ctrl.hdr.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET;
271	memcpy(ctrl.data, &nb_queues, sizeof(uint16_t));
272
273	dlen[0] = sizeof(uint16_t);
274
275	ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
276	if (ret) {
277		PMD_INIT_LOG(ERR, "Multiqueue configured but send command "
278			  "failed, this is too late now...");
279		return -EINVAL;
280	}
281
282	return 0;
283}
284
285static void
286virtio_dev_queue_release(void *queue __rte_unused)
287{
288	/* do nothing */
289}
290
291static int
292virtio_get_queue_type(struct virtio_hw *hw, uint16_t vtpci_queue_idx)
293{
294	if (vtpci_queue_idx == hw->max_queue_pairs * 2)
295		return VTNET_CQ;
296	else if (vtpci_queue_idx % 2 == 0)
297		return VTNET_RQ;
298	else
299		return VTNET_TQ;
300}
301
302static uint16_t
303virtio_get_nr_vq(struct virtio_hw *hw)
304{
305	uint16_t nr_vq = hw->max_queue_pairs * 2;
306
307	if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VQ))
308		nr_vq += 1;
309
310	return nr_vq;
311}
312
313static void
314virtio_init_vring(struct virtqueue *vq)
315{
316	int size = vq->vq_nentries;
317	struct vring *vr = &vq->vq_ring;
318	uint8_t *ring_mem = vq->vq_ring_virt_mem;
319
320	PMD_INIT_FUNC_TRACE();
321
322	/*
323	 * Reinitialise since virtio port might have been stopped and restarted
324	 */
325	memset(ring_mem, 0, vq->vq_ring_size);
326	vring_init(vr, size, ring_mem, VIRTIO_PCI_VRING_ALIGN);
327	vq->vq_used_cons_idx = 0;
328	vq->vq_desc_head_idx = 0;
329	vq->vq_avail_idx = 0;
330	vq->vq_desc_tail_idx = (uint16_t)(vq->vq_nentries - 1);
331	vq->vq_free_cnt = vq->vq_nentries;
332	memset(vq->vq_descx, 0, sizeof(struct vq_desc_extra) * vq->vq_nentries);
333
334	vring_desc_init(vr->desc, size);
335
336	/*
337	 * Disable device(host) interrupting guest
338	 */
339	virtqueue_disable_intr(vq);
340}
341
342static int
343virtio_init_queue(struct rte_eth_dev *dev, uint16_t vtpci_queue_idx)
344{
345	char vq_name[VIRTQUEUE_MAX_NAME_SZ];
346	char vq_hdr_name[VIRTQUEUE_MAX_NAME_SZ];
347	const struct rte_memzone *mz = NULL, *hdr_mz = NULL;
348	unsigned int vq_size, size;
349	struct virtio_hw *hw = dev->data->dev_private;
350	struct virtnet_rx *rxvq = NULL;
351	struct virtnet_tx *txvq = NULL;
352	struct virtnet_ctl *cvq = NULL;
353	struct virtqueue *vq;
354	size_t sz_hdr_mz = 0;
355	void *sw_ring = NULL;
356	int queue_type = virtio_get_queue_type(hw, vtpci_queue_idx);
357	int ret;
358
359	PMD_INIT_LOG(DEBUG, "setting up queue: %u", vtpci_queue_idx);
360
361	/*
362	 * Read the virtqueue size from the Queue Size field
363	 * Always power of 2 and if 0 virtqueue does not exist
364	 */
365	vq_size = VTPCI_OPS(hw)->get_queue_num(hw, vtpci_queue_idx);
366	PMD_INIT_LOG(DEBUG, "vq_size: %u", vq_size);
367	if (vq_size == 0) {
368		PMD_INIT_LOG(ERR, "virtqueue does not exist");
369		return -EINVAL;
370	}
371
372	if (!rte_is_power_of_2(vq_size)) {
373		PMD_INIT_LOG(ERR, "virtqueue size is not powerof 2");
374		return -EINVAL;
375	}
376
377	snprintf(vq_name, sizeof(vq_name), "port%d_vq%d",
378		 dev->data->port_id, vtpci_queue_idx);
379
380	size = RTE_ALIGN_CEIL(sizeof(*vq) +
381				vq_size * sizeof(struct vq_desc_extra),
382				RTE_CACHE_LINE_SIZE);
383	if (queue_type == VTNET_TQ) {
384		/*
385		 * For each xmit packet, allocate a virtio_net_hdr
386		 * and indirect ring elements
387		 */
388		sz_hdr_mz = vq_size * sizeof(struct virtio_tx_region);
389	} else if (queue_type == VTNET_CQ) {
390		/* Allocate a page for control vq command, data and status */
391		sz_hdr_mz = PAGE_SIZE;
392	}
393
394	vq = rte_zmalloc_socket(vq_name, size, RTE_CACHE_LINE_SIZE,
395				SOCKET_ID_ANY);
396	if (vq == NULL) {
397		PMD_INIT_LOG(ERR, "can not allocate vq");
398		return -ENOMEM;
399	}
400	hw->vqs[vtpci_queue_idx] = vq;
401
402	vq->hw = hw;
403	vq->vq_queue_index = vtpci_queue_idx;
404	vq->vq_nentries = vq_size;
405
406	/*
407	 * Reserve a memzone for vring elements
408	 */
409	size = vring_size(vq_size, VIRTIO_PCI_VRING_ALIGN);
410	vq->vq_ring_size = RTE_ALIGN_CEIL(size, VIRTIO_PCI_VRING_ALIGN);
411	PMD_INIT_LOG(DEBUG, "vring_size: %d, rounded_vring_size: %d",
412		     size, vq->vq_ring_size);
413
414	mz = rte_memzone_reserve_aligned(vq_name, vq->vq_ring_size,
415					 SOCKET_ID_ANY,
416					 0, VIRTIO_PCI_VRING_ALIGN);
417	if (mz == NULL) {
418		if (rte_errno == EEXIST)
419			mz = rte_memzone_lookup(vq_name);
420		if (mz == NULL) {
421			ret = -ENOMEM;
422			goto fail_q_alloc;
423		}
424	}
425
426	memset(mz->addr, 0, sizeof(mz->len));
427
428	vq->vq_ring_mem = mz->phys_addr;
429	vq->vq_ring_virt_mem = mz->addr;
430	PMD_INIT_LOG(DEBUG, "vq->vq_ring_mem:      0x%" PRIx64,
431		     (uint64_t)mz->phys_addr);
432	PMD_INIT_LOG(DEBUG, "vq->vq_ring_virt_mem: 0x%" PRIx64,
433		     (uint64_t)(uintptr_t)mz->addr);
434
435	virtio_init_vring(vq);
436
437	if (sz_hdr_mz) {
438		snprintf(vq_hdr_name, sizeof(vq_hdr_name), "port%d_vq%d_hdr",
439			 dev->data->port_id, vtpci_queue_idx);
440		hdr_mz = rte_memzone_reserve_aligned(vq_hdr_name, sz_hdr_mz,
441						     SOCKET_ID_ANY, 0,
442						     RTE_CACHE_LINE_SIZE);
443		if (hdr_mz == NULL) {
444			if (rte_errno == EEXIST)
445				hdr_mz = rte_memzone_lookup(vq_hdr_name);
446			if (hdr_mz == NULL) {
447				ret = -ENOMEM;
448				goto fail_q_alloc;
449			}
450		}
451	}
452
453	if (queue_type == VTNET_RQ) {
454		size_t sz_sw = (RTE_PMD_VIRTIO_RX_MAX_BURST + vq_size) *
455			       sizeof(vq->sw_ring[0]);
456
457		sw_ring = rte_zmalloc_socket("sw_ring", sz_sw,
458				RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
459		if (!sw_ring) {
460			PMD_INIT_LOG(ERR, "can not allocate RX soft ring");
461			ret = -ENOMEM;
462			goto fail_q_alloc;
463		}
464
465		vq->sw_ring = sw_ring;
466		rxvq = &vq->rxq;
467		rxvq->vq = vq;
468		rxvq->port_id = dev->data->port_id;
469		rxvq->mz = mz;
470	} else if (queue_type == VTNET_TQ) {
471		txvq = &vq->txq;
472		txvq->vq = vq;
473		txvq->port_id = dev->data->port_id;
474		txvq->mz = mz;
475		txvq->virtio_net_hdr_mz = hdr_mz;
476		txvq->virtio_net_hdr_mem = hdr_mz->phys_addr;
477	} else if (queue_type == VTNET_CQ) {
478		cvq = &vq->cq;
479		cvq->vq = vq;
480		cvq->mz = mz;
481		cvq->virtio_net_hdr_mz = hdr_mz;
482		cvq->virtio_net_hdr_mem = hdr_mz->phys_addr;
483		memset(cvq->virtio_net_hdr_mz->addr, 0, PAGE_SIZE);
484
485		hw->cvq = cvq;
486	}
487
488	/* For virtio_user case (that is when hw->dev is NULL), we use
489	 * virtual address. And we need properly set _offset_, please see
490	 * VIRTIO_MBUF_DATA_DMA_ADDR in virtqueue.h for more information.
491	 */
492	if (!hw->virtio_user_dev)
493		vq->offset = offsetof(struct rte_mbuf, buf_physaddr);
494	else {
495		vq->vq_ring_mem = (uintptr_t)mz->addr;
496		vq->offset = offsetof(struct rte_mbuf, buf_addr);
497		if (queue_type == VTNET_TQ)
498			txvq->virtio_net_hdr_mem = (uintptr_t)hdr_mz->addr;
499		else if (queue_type == VTNET_CQ)
500			cvq->virtio_net_hdr_mem = (uintptr_t)hdr_mz->addr;
501	}
502
503	if (queue_type == VTNET_TQ) {
504		struct virtio_tx_region *txr;
505		unsigned int i;
506
507		txr = hdr_mz->addr;
508		memset(txr, 0, vq_size * sizeof(*txr));
509		for (i = 0; i < vq_size; i++) {
510			struct vring_desc *start_dp = txr[i].tx_indir;
511
512			vring_desc_init(start_dp, RTE_DIM(txr[i].tx_indir));
513
514			/* first indirect descriptor is always the tx header */
515			start_dp->addr = txvq->virtio_net_hdr_mem
516				+ i * sizeof(*txr)
517				+ offsetof(struct virtio_tx_region, tx_hdr);
518
519			start_dp->len = hw->vtnet_hdr_size;
520			start_dp->flags = VRING_DESC_F_NEXT;
521		}
522	}
523
524	if (VTPCI_OPS(hw)->setup_queue(hw, vq) < 0) {
525		PMD_INIT_LOG(ERR, "setup_queue failed");
526		return -EINVAL;
527	}
528
529	return 0;
530
531fail_q_alloc:
532	rte_free(sw_ring);
533	rte_memzone_free(hdr_mz);
534	rte_memzone_free(mz);
535	rte_free(vq);
536
537	return ret;
538}
539
540static void
541virtio_free_queues(struct virtio_hw *hw)
542{
543	uint16_t nr_vq = virtio_get_nr_vq(hw);
544	struct virtqueue *vq;
545	int queue_type;
546	uint16_t i;
547
548	for (i = 0; i < nr_vq; i++) {
549		vq = hw->vqs[i];
550		if (!vq)
551			continue;
552
553		queue_type = virtio_get_queue_type(hw, i);
554		if (queue_type == VTNET_RQ) {
555			rte_free(vq->sw_ring);
556			rte_memzone_free(vq->rxq.mz);
557		} else if (queue_type == VTNET_TQ) {
558			rte_memzone_free(vq->txq.mz);
559			rte_memzone_free(vq->txq.virtio_net_hdr_mz);
560		} else {
561			rte_memzone_free(vq->cq.mz);
562			rte_memzone_free(vq->cq.virtio_net_hdr_mz);
563		}
564
565		rte_free(vq);
566	}
567
568	rte_free(hw->vqs);
569}
570
571static int
572virtio_alloc_queues(struct rte_eth_dev *dev)
573{
574	struct virtio_hw *hw = dev->data->dev_private;
575	uint16_t nr_vq = virtio_get_nr_vq(hw);
576	uint16_t i;
577	int ret;
578
579	hw->vqs = rte_zmalloc(NULL, sizeof(struct virtqueue *) * nr_vq, 0);
580	if (!hw->vqs) {
581		PMD_INIT_LOG(ERR, "failed to allocate vqs");
582		return -ENOMEM;
583	}
584
585	for (i = 0; i < nr_vq; i++) {
586		ret = virtio_init_queue(dev, i);
587		if (ret < 0) {
588			virtio_free_queues(hw);
589			return ret;
590		}
591	}
592
593	return 0;
594}
595
596static void virtio_queues_unbind_intr(struct rte_eth_dev *dev);
597
598static void
599virtio_dev_close(struct rte_eth_dev *dev)
600{
601	struct virtio_hw *hw = dev->data->dev_private;
602	struct rte_intr_conf *intr_conf = &dev->data->dev_conf.intr_conf;
603
604	PMD_INIT_LOG(DEBUG, "virtio_dev_close");
605
606	/* reset the NIC */
607	if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
608		VTPCI_OPS(hw)->set_config_irq(hw, VIRTIO_MSI_NO_VECTOR);
609	if (intr_conf->rxq)
610		virtio_queues_unbind_intr(dev);
611
612	if (intr_conf->lsc || intr_conf->rxq) {
613		rte_intr_disable(dev->intr_handle);
614		rte_intr_efd_disable(dev->intr_handle);
615		rte_free(dev->intr_handle->intr_vec);
616		dev->intr_handle->intr_vec = NULL;
617	}
618
619	vtpci_reset(hw);
620	virtio_dev_free_mbufs(dev);
621	virtio_free_queues(hw);
622}
623
624static void
625virtio_dev_promiscuous_enable(struct rte_eth_dev *dev)
626{
627	struct virtio_hw *hw = dev->data->dev_private;
628	struct virtio_pmd_ctrl ctrl;
629	int dlen[1];
630	int ret;
631
632	if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
633		PMD_INIT_LOG(INFO, "host does not support rx control");
634		return;
635	}
636
637	ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
638	ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_PROMISC;
639	ctrl.data[0] = 1;
640	dlen[0] = 1;
641
642	ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
643	if (ret)
644		PMD_INIT_LOG(ERR, "Failed to enable promisc");
645}
646
647static void
648virtio_dev_promiscuous_disable(struct rte_eth_dev *dev)
649{
650	struct virtio_hw *hw = dev->data->dev_private;
651	struct virtio_pmd_ctrl ctrl;
652	int dlen[1];
653	int ret;
654
655	if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
656		PMD_INIT_LOG(INFO, "host does not support rx control");
657		return;
658	}
659
660	ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
661	ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_PROMISC;
662	ctrl.data[0] = 0;
663	dlen[0] = 1;
664
665	ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
666	if (ret)
667		PMD_INIT_LOG(ERR, "Failed to disable promisc");
668}
669
670static void
671virtio_dev_allmulticast_enable(struct rte_eth_dev *dev)
672{
673	struct virtio_hw *hw = dev->data->dev_private;
674	struct virtio_pmd_ctrl ctrl;
675	int dlen[1];
676	int ret;
677
678	if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
679		PMD_INIT_LOG(INFO, "host does not support rx control");
680		return;
681	}
682
683	ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
684	ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_ALLMULTI;
685	ctrl.data[0] = 1;
686	dlen[0] = 1;
687
688	ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
689	if (ret)
690		PMD_INIT_LOG(ERR, "Failed to enable allmulticast");
691}
692
693static void
694virtio_dev_allmulticast_disable(struct rte_eth_dev *dev)
695{
696	struct virtio_hw *hw = dev->data->dev_private;
697	struct virtio_pmd_ctrl ctrl;
698	int dlen[1];
699	int ret;
700
701	if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_RX)) {
702		PMD_INIT_LOG(INFO, "host does not support rx control");
703		return;
704	}
705
706	ctrl.hdr.class = VIRTIO_NET_CTRL_RX;
707	ctrl.hdr.cmd = VIRTIO_NET_CTRL_RX_ALLMULTI;
708	ctrl.data[0] = 0;
709	dlen[0] = 1;
710
711	ret = virtio_send_command(hw->cvq, &ctrl, dlen, 1);
712	if (ret)
713		PMD_INIT_LOG(ERR, "Failed to disable allmulticast");
714}
715
716#define VLAN_TAG_LEN           4    /* 802.3ac tag (not DMA'd) */
717static int
718virtio_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
719{
720	struct virtio_hw *hw = dev->data->dev_private;
721	uint32_t ether_hdr_len = ETHER_HDR_LEN + VLAN_TAG_LEN +
722				 hw->vtnet_hdr_size;
723	uint32_t frame_size = mtu + ether_hdr_len;
724
725	if (mtu < ETHER_MIN_MTU || frame_size > VIRTIO_MAX_RX_PKTLEN) {
726		PMD_INIT_LOG(ERR, "MTU should be between %d and %d",
727			ETHER_MIN_MTU, VIRTIO_MAX_RX_PKTLEN - ether_hdr_len);
728		return -EINVAL;
729	}
730	return 0;
731}
732
733static int
734virtio_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
735{
736	struct virtnet_rx *rxvq = dev->data->rx_queues[queue_id];
737	struct virtqueue *vq = rxvq->vq;
738
739	virtqueue_enable_intr(vq);
740	return 0;
741}
742
743static int
744virtio_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
745{
746	struct virtnet_rx *rxvq = dev->data->rx_queues[queue_id];
747	struct virtqueue *vq = rxvq->vq;
748
749	virtqueue_disable_intr(vq);
750	return 0;
751}
752
753/*
754 * dev_ops for virtio, bare necessities for basic operation
755 */
756static const struct eth_dev_ops virtio_eth_dev_ops = {
757	.dev_configure           = virtio_dev_configure,
758	.dev_start               = virtio_dev_start,
759	.dev_stop                = virtio_dev_stop,
760	.dev_close               = virtio_dev_close,
761	.promiscuous_enable      = virtio_dev_promiscuous_enable,
762	.promiscuous_disable     = virtio_dev_promiscuous_disable,
763	.allmulticast_enable     = virtio_dev_allmulticast_enable,
764	.allmulticast_disable    = virtio_dev_allmulticast_disable,
765	.mtu_set                 = virtio_mtu_set,
766	.dev_infos_get           = virtio_dev_info_get,
767	.stats_get               = virtio_dev_stats_get,
768	.xstats_get              = virtio_dev_xstats_get,
769	.xstats_get_names        = virtio_dev_xstats_get_names,
770	.stats_reset             = virtio_dev_stats_reset,
771	.xstats_reset            = virtio_dev_stats_reset,
772	.link_update             = virtio_dev_link_update,
773	.rx_queue_setup          = virtio_dev_rx_queue_setup,
774	.rx_queue_intr_enable    = virtio_dev_rx_queue_intr_enable,
775	.rx_queue_intr_disable   = virtio_dev_rx_queue_intr_disable,
776	.rx_queue_release        = virtio_dev_queue_release,
777	.rx_descriptor_done      = virtio_dev_rx_queue_done,
778	.tx_queue_setup          = virtio_dev_tx_queue_setup,
779	.tx_queue_release        = virtio_dev_queue_release,
780	/* collect stats per queue */
781	.queue_stats_mapping_set = virtio_dev_queue_stats_mapping_set,
782	.vlan_filter_set         = virtio_vlan_filter_set,
783	.mac_addr_add            = virtio_mac_addr_add,
784	.mac_addr_remove         = virtio_mac_addr_remove,
785	.mac_addr_set            = virtio_mac_addr_set,
786};
787
788static inline int
789virtio_dev_atomic_read_link_status(struct rte_eth_dev *dev,
790				struct rte_eth_link *link)
791{
792	struct rte_eth_link *dst = link;
793	struct rte_eth_link *src = &(dev->data->dev_link);
794
795	if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
796			*(uint64_t *)src) == 0)
797		return -1;
798
799	return 0;
800}
801
802/**
803 * Atomically writes the link status information into global
804 * structure rte_eth_dev.
805 *
806 * @param dev
807 *   - Pointer to the structure rte_eth_dev to read from.
808 *   - Pointer to the buffer to be saved with the link status.
809 *
810 * @return
811 *   - On success, zero.
812 *   - On failure, negative value.
813 */
814static inline int
815virtio_dev_atomic_write_link_status(struct rte_eth_dev *dev,
816		struct rte_eth_link *link)
817{
818	struct rte_eth_link *dst = &(dev->data->dev_link);
819	struct rte_eth_link *src = link;
820
821	if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
822					*(uint64_t *)src) == 0)
823		return -1;
824
825	return 0;
826}
827
828static void
829virtio_update_stats(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
830{
831	unsigned i;
832
833	for (i = 0; i < dev->data->nb_tx_queues; i++) {
834		const struct virtnet_tx *txvq = dev->data->tx_queues[i];
835		if (txvq == NULL)
836			continue;
837
838		stats->opackets += txvq->stats.packets;
839		stats->obytes += txvq->stats.bytes;
840		stats->oerrors += txvq->stats.errors;
841
842		if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
843			stats->q_opackets[i] = txvq->stats.packets;
844			stats->q_obytes[i] = txvq->stats.bytes;
845		}
846	}
847
848	for (i = 0; i < dev->data->nb_rx_queues; i++) {
849		const struct virtnet_rx *rxvq = dev->data->rx_queues[i];
850		if (rxvq == NULL)
851			continue;
852
853		stats->ipackets += rxvq->stats.packets;
854		stats->ibytes += rxvq->stats.bytes;
855		stats->ierrors += rxvq->stats.errors;
856
857		if (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) {
858			stats->q_ipackets[i] = rxvq->stats.packets;
859			stats->q_ibytes[i] = rxvq->stats.bytes;
860		}
861	}
862
863	stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed;
864}
865
866static int virtio_dev_xstats_get_names(struct rte_eth_dev *dev,
867				       struct rte_eth_xstat_name *xstats_names,
868				       __rte_unused unsigned limit)
869{
870	unsigned i;
871	unsigned count = 0;
872	unsigned t;
873
874	unsigned nstats = dev->data->nb_tx_queues * VIRTIO_NB_TXQ_XSTATS +
875		dev->data->nb_rx_queues * VIRTIO_NB_RXQ_XSTATS;
876
877	if (xstats_names != NULL) {
878		/* Note: limit checked in rte_eth_xstats_names() */
879
880		for (i = 0; i < dev->data->nb_rx_queues; i++) {
881			struct virtqueue *rxvq = dev->data->rx_queues[i];
882			if (rxvq == NULL)
883				continue;
884			for (t = 0; t < VIRTIO_NB_RXQ_XSTATS; t++) {
885				snprintf(xstats_names[count].name,
886					sizeof(xstats_names[count].name),
887					"rx_q%u_%s", i,
888					rte_virtio_rxq_stat_strings[t].name);
889				count++;
890			}
891		}
892
893		for (i = 0; i < dev->data->nb_tx_queues; i++) {
894			struct virtqueue *txvq = dev->data->tx_queues[i];
895			if (txvq == NULL)
896				continue;
897			for (t = 0; t < VIRTIO_NB_TXQ_XSTATS; t++) {
898				snprintf(xstats_names[count].name,
899					sizeof(xstats_names[count].name),
900					"tx_q%u_%s", i,
901					rte_virtio_txq_stat_strings[t].name);
902				count++;
903			}
904		}
905		return count;
906	}
907	return nstats;
908}
909
910static int
911virtio_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
912		      unsigned n)
913{
914	unsigned i;
915	unsigned count = 0;
916
917	unsigned nstats = dev->data->nb_tx_queues * VIRTIO_NB_TXQ_XSTATS +
918		dev->data->nb_rx_queues * VIRTIO_NB_RXQ_XSTATS;
919
920	if (n < nstats)
921		return nstats;
922
923	for (i = 0; i < dev->data->nb_rx_queues; i++) {
924		struct virtnet_rx *rxvq = dev->data->rx_queues[i];
925
926		if (rxvq == NULL)
927			continue;
928
929		unsigned t;
930
931		for (t = 0; t < VIRTIO_NB_RXQ_XSTATS; t++) {
932			xstats[count].value = *(uint64_t *)(((char *)rxvq) +
933				rte_virtio_rxq_stat_strings[t].offset);
934			xstats[count].id = count;
935			count++;
936		}
937	}
938
939	for (i = 0; i < dev->data->nb_tx_queues; i++) {
940		struct virtnet_tx *txvq = dev->data->tx_queues[i];
941
942		if (txvq == NULL)
943			continue;
944
945		unsigned t;
946
947		for (t = 0; t < VIRTIO_NB_TXQ_XSTATS; t++) {
948			xstats[count].value = *(uint64_t *)(((char *)txvq) +
949				rte_virtio_txq_stat_strings[t].offset);
950			xstats[count].id = count;
951			count++;
952		}
953	}
954
955	return count;
956}
957
958static void
959virtio_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
960{
961	virtio_update_stats(dev, stats);
962}
963
964static void
965virtio_dev_stats_reset(struct rte_eth_dev *dev)
966{
967	unsigned int i;
968
969	for (i = 0; i < dev->data->nb_tx_queues; i++) {
970		struct virtnet_tx *txvq = dev->data->tx_queues[i];
971		if (txvq == NULL)
972			continue;
973
974		txvq->stats.packets = 0;
975		txvq->stats.bytes = 0;
976		txvq->stats.errors = 0;
977		txvq->stats.multicast = 0;
978		txvq->stats.broadcast = 0;
979		memset(txvq->stats.size_bins, 0,
980		       sizeof(txvq->stats.size_bins[0]) * 8);
981	}
982
983	for (i = 0; i < dev->data->nb_rx_queues; i++) {
984		struct virtnet_rx *rxvq = dev->data->rx_queues[i];
985		if (rxvq == NULL)
986			continue;
987
988		rxvq->stats.packets = 0;
989		rxvq->stats.bytes = 0;
990		rxvq->stats.errors = 0;
991		rxvq->stats.multicast = 0;
992		rxvq->stats.broadcast = 0;
993		memset(rxvq->stats.size_bins, 0,
994		       sizeof(rxvq->stats.size_bins[0]) * 8);
995	}
996}
997
998static void
999virtio_set_hwaddr(struct virtio_hw *hw)
1000{
1001	vtpci_write_dev_config(hw,
1002			offsetof(struct virtio_net_config, mac),
1003			&hw->mac_addr, ETHER_ADDR_LEN);
1004}
1005
1006static void
1007virtio_get_hwaddr(struct virtio_hw *hw)
1008{
1009	if (vtpci_with_feature(hw, VIRTIO_NET_F_MAC)) {
1010		vtpci_read_dev_config(hw,
1011			offsetof(struct virtio_net_config, mac),
1012			&hw->mac_addr, ETHER_ADDR_LEN);
1013	} else {
1014		eth_random_addr(&hw->mac_addr[0]);
1015		virtio_set_hwaddr(hw);
1016	}
1017}
1018
1019static void
1020virtio_mac_table_set(struct virtio_hw *hw,
1021		     const struct virtio_net_ctrl_mac *uc,
1022		     const struct virtio_net_ctrl_mac *mc)
1023{
1024	struct virtio_pmd_ctrl ctrl;
1025	int err, len[2];
1026
1027	if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
1028		PMD_DRV_LOG(INFO, "host does not support mac table");
1029		return;
1030	}
1031
1032	ctrl.hdr.class = VIRTIO_NET_CTRL_MAC;
1033	ctrl.hdr.cmd = VIRTIO_NET_CTRL_MAC_TABLE_SET;
1034
1035	len[0] = uc->entries * ETHER_ADDR_LEN + sizeof(uc->entries);
1036	memcpy(ctrl.data, uc, len[0]);
1037
1038	len[1] = mc->entries * ETHER_ADDR_LEN + sizeof(mc->entries);
1039	memcpy(ctrl.data + len[0], mc, len[1]);
1040
1041	err = virtio_send_command(hw->cvq, &ctrl, len, 2);
1042	if (err != 0)
1043		PMD_DRV_LOG(NOTICE, "mac table set failed: %d", err);
1044}
1045
1046static void
1047virtio_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
1048		    uint32_t index, uint32_t vmdq __rte_unused)
1049{
1050	struct virtio_hw *hw = dev->data->dev_private;
1051	const struct ether_addr *addrs = dev->data->mac_addrs;
1052	unsigned int i;
1053	struct virtio_net_ctrl_mac *uc, *mc;
1054
1055	if (index >= VIRTIO_MAX_MAC_ADDRS) {
1056		PMD_DRV_LOG(ERR, "mac address index %u out of range", index);
1057		return;
1058	}
1059
1060	uc = alloca(VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN + sizeof(uc->entries));
1061	uc->entries = 0;
1062	mc = alloca(VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN + sizeof(mc->entries));
1063	mc->entries = 0;
1064
1065	for (i = 0; i < VIRTIO_MAX_MAC_ADDRS; i++) {
1066		const struct ether_addr *addr
1067			= (i == index) ? mac_addr : addrs + i;
1068		struct virtio_net_ctrl_mac *tbl
1069			= is_multicast_ether_addr(addr) ? mc : uc;
1070
1071		memcpy(&tbl->macs[tbl->entries++], addr, ETHER_ADDR_LEN);
1072	}
1073
1074	virtio_mac_table_set(hw, uc, mc);
1075}
1076
1077static void
1078virtio_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
1079{
1080	struct virtio_hw *hw = dev->data->dev_private;
1081	struct ether_addr *addrs = dev->data->mac_addrs;
1082	struct virtio_net_ctrl_mac *uc, *mc;
1083	unsigned int i;
1084
1085	if (index >= VIRTIO_MAX_MAC_ADDRS) {
1086		PMD_DRV_LOG(ERR, "mac address index %u out of range", index);
1087		return;
1088	}
1089
1090	uc = alloca(VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN + sizeof(uc->entries));
1091	uc->entries = 0;
1092	mc = alloca(VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN + sizeof(mc->entries));
1093	mc->entries = 0;
1094
1095	for (i = 0; i < VIRTIO_MAX_MAC_ADDRS; i++) {
1096		struct virtio_net_ctrl_mac *tbl;
1097
1098		if (i == index || is_zero_ether_addr(addrs + i))
1099			continue;
1100
1101		tbl = is_multicast_ether_addr(addrs + i) ? mc : uc;
1102		memcpy(&tbl->macs[tbl->entries++], addrs + i, ETHER_ADDR_LEN);
1103	}
1104
1105	virtio_mac_table_set(hw, uc, mc);
1106}
1107
1108static void
1109virtio_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
1110{
1111	struct virtio_hw *hw = dev->data->dev_private;
1112
1113	memcpy(hw->mac_addr, mac_addr, ETHER_ADDR_LEN);
1114
1115	/* Use atomic update if available */
1116	if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_MAC_ADDR)) {
1117		struct virtio_pmd_ctrl ctrl;
1118		int len = ETHER_ADDR_LEN;
1119
1120		ctrl.hdr.class = VIRTIO_NET_CTRL_MAC;
1121		ctrl.hdr.cmd = VIRTIO_NET_CTRL_MAC_ADDR_SET;
1122
1123		memcpy(ctrl.data, mac_addr, ETHER_ADDR_LEN);
1124		virtio_send_command(hw->cvq, &ctrl, &len, 1);
1125	} else if (vtpci_with_feature(hw, VIRTIO_NET_F_MAC))
1126		virtio_set_hwaddr(hw);
1127}
1128
1129static int
1130virtio_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1131{
1132	struct virtio_hw *hw = dev->data->dev_private;
1133	struct virtio_pmd_ctrl ctrl;
1134	int len;
1135
1136	if (!vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN))
1137		return -ENOTSUP;
1138
1139	ctrl.hdr.class = VIRTIO_NET_CTRL_VLAN;
1140	ctrl.hdr.cmd = on ? VIRTIO_NET_CTRL_VLAN_ADD : VIRTIO_NET_CTRL_VLAN_DEL;
1141	memcpy(ctrl.data, &vlan_id, sizeof(vlan_id));
1142	len = sizeof(vlan_id);
1143
1144	return virtio_send_command(hw->cvq, &ctrl, &len, 1);
1145}
1146
1147static int
1148virtio_negotiate_features(struct virtio_hw *hw, uint64_t req_features)
1149{
1150	uint64_t host_features;
1151
1152	/* Prepare guest_features: feature that driver wants to support */
1153	PMD_INIT_LOG(DEBUG, "guest_features before negotiate = %" PRIx64,
1154		req_features);
1155
1156	/* Read device(host) feature bits */
1157	host_features = VTPCI_OPS(hw)->get_features(hw);
1158	PMD_INIT_LOG(DEBUG, "host_features before negotiate = %" PRIx64,
1159		host_features);
1160
1161	/*
1162	 * Negotiate features: Subset of device feature bits are written back
1163	 * guest feature bits.
1164	 */
1165	hw->guest_features = req_features;
1166	hw->guest_features = vtpci_negotiate_features(hw, host_features);
1167	PMD_INIT_LOG(DEBUG, "features after negotiate = %" PRIx64,
1168		hw->guest_features);
1169
1170	if (hw->modern) {
1171		if (!vtpci_with_feature(hw, VIRTIO_F_VERSION_1)) {
1172			PMD_INIT_LOG(ERR,
1173				"VIRTIO_F_VERSION_1 features is not enabled.");
1174			return -1;
1175		}
1176		vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_FEATURES_OK);
1177		if (!(vtpci_get_status(hw) & VIRTIO_CONFIG_STATUS_FEATURES_OK)) {
1178			PMD_INIT_LOG(ERR,
1179				"failed to set FEATURES_OK status!");
1180			return -1;
1181		}
1182	}
1183
1184	hw->req_guest_features = req_features;
1185
1186	return 0;
1187}
1188
1189/*
1190 * Process Virtio Config changed interrupt and call the callback
1191 * if link state changed.
1192 */
1193static void
1194virtio_interrupt_handler(struct rte_intr_handle *handle,
1195			 void *param)
1196{
1197	struct rte_eth_dev *dev = param;
1198	struct virtio_hw *hw = dev->data->dev_private;
1199	uint8_t isr;
1200
1201	/* Read interrupt status which clears interrupt */
1202	isr = vtpci_isr(hw);
1203	PMD_DRV_LOG(INFO, "interrupt status = %#x", isr);
1204
1205	if (rte_intr_enable(handle) < 0)
1206		PMD_DRV_LOG(ERR, "interrupt enable failed");
1207
1208	if (isr & VIRTIO_PCI_ISR_CONFIG) {
1209		if (virtio_dev_link_update(dev, 0) == 0)
1210			_rte_eth_dev_callback_process(dev,
1211						      RTE_ETH_EVENT_INTR_LSC, NULL);
1212	}
1213
1214}
1215
1216static void
1217rx_func_get(struct rte_eth_dev *eth_dev)
1218{
1219	struct virtio_hw *hw = eth_dev->data->dev_private;
1220	if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF))
1221		eth_dev->rx_pkt_burst = &virtio_recv_mergeable_pkts;
1222	else
1223		eth_dev->rx_pkt_burst = &virtio_recv_pkts;
1224}
1225
1226/* Only support 1:1 queue/interrupt mapping so far.
1227 * TODO: support n:1 queue/interrupt mapping when there are limited number of
1228 * interrupt vectors (<N+1).
1229 */
1230static int
1231virtio_queues_bind_intr(struct rte_eth_dev *dev)
1232{
1233	uint32_t i;
1234	struct virtio_hw *hw = dev->data->dev_private;
1235
1236	PMD_INIT_LOG(INFO, "queue/interrupt binding");
1237	for (i = 0; i < dev->data->nb_rx_queues; ++i) {
1238		dev->intr_handle->intr_vec[i] = i + 1;
1239		if (VTPCI_OPS(hw)->set_queue_irq(hw, hw->vqs[i * 2], i + 1) ==
1240						 VIRTIO_MSI_NO_VECTOR) {
1241			PMD_DRV_LOG(ERR, "failed to set queue vector");
1242			return -EBUSY;
1243		}
1244	}
1245
1246	return 0;
1247}
1248
1249static void
1250virtio_queues_unbind_intr(struct rte_eth_dev *dev)
1251{
1252	uint32_t i;
1253	struct virtio_hw *hw = dev->data->dev_private;
1254
1255	PMD_INIT_LOG(INFO, "queue/interrupt unbinding");
1256	for (i = 0; i < dev->data->nb_rx_queues; ++i)
1257		VTPCI_OPS(hw)->set_queue_irq(hw,
1258					     hw->vqs[i * VTNET_CQ],
1259					     VIRTIO_MSI_NO_VECTOR);
1260}
1261
1262static int
1263virtio_configure_intr(struct rte_eth_dev *dev)
1264{
1265	struct virtio_hw *hw = dev->data->dev_private;
1266
1267	if (!rte_intr_cap_multiple(dev->intr_handle)) {
1268		PMD_INIT_LOG(ERR, "Multiple intr vector not supported");
1269		return -ENOTSUP;
1270	}
1271
1272	if (rte_intr_efd_enable(dev->intr_handle, dev->data->nb_rx_queues)) {
1273		PMD_INIT_LOG(ERR, "Fail to create eventfd");
1274		return -1;
1275	}
1276
1277	if (!dev->intr_handle->intr_vec) {
1278		dev->intr_handle->intr_vec =
1279			rte_zmalloc("intr_vec",
1280				    hw->max_queue_pairs * sizeof(int), 0);
1281		if (!dev->intr_handle->intr_vec) {
1282			PMD_INIT_LOG(ERR, "Failed to allocate %u rxq vectors",
1283				     hw->max_queue_pairs);
1284			return -ENOMEM;
1285		}
1286	}
1287
1288	/* Re-register callback to update max_intr */
1289	rte_intr_callback_unregister(dev->intr_handle,
1290				     virtio_interrupt_handler,
1291				     dev);
1292	rte_intr_callback_register(dev->intr_handle,
1293				   virtio_interrupt_handler,
1294				   dev);
1295
1296	/* DO NOT try to remove this! This function will enable msix, or QEMU
1297	 * will encounter SIGSEGV when DRIVER_OK is sent.
1298	 * And for legacy devices, this should be done before queue/vec binding
1299	 * to change the config size from 20 to 24, or VIRTIO_MSI_QUEUE_VECTOR
1300	 * (22) will be ignored.
1301	 */
1302	if (rte_intr_enable(dev->intr_handle) < 0) {
1303		PMD_DRV_LOG(ERR, "interrupt enable failed");
1304		return -1;
1305	}
1306
1307	if (virtio_queues_bind_intr(dev) < 0) {
1308		PMD_INIT_LOG(ERR, "Failed to bind queue/interrupt");
1309		return -1;
1310	}
1311
1312	return 0;
1313}
1314
1315/* reset device and renegotiate features if needed */
1316static int
1317virtio_init_device(struct rte_eth_dev *eth_dev, uint64_t req_features)
1318{
1319	struct virtio_hw *hw = eth_dev->data->dev_private;
1320	struct virtio_net_config *config;
1321	struct virtio_net_config local_config;
1322	struct rte_pci_device *pci_dev = NULL;
1323	int ret;
1324
1325	/* Reset the device although not necessary at startup */
1326	vtpci_reset(hw);
1327
1328	/* Tell the host we've noticed this device. */
1329	vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_ACK);
1330
1331	/* Tell the host we've known how to drive the device. */
1332	vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER);
1333	if (virtio_negotiate_features(hw, req_features) < 0)
1334		return -1;
1335
1336	if (eth_dev->device) {
1337		pci_dev = RTE_DEV_TO_PCI(eth_dev->device);
1338		rte_eth_copy_pci_info(eth_dev, pci_dev);
1339	}
1340
1341	/* If host does not support status then disable LSC */
1342	if (!vtpci_with_feature(hw, VIRTIO_NET_F_STATUS))
1343		eth_dev->data->dev_flags &= ~RTE_ETH_DEV_INTR_LSC;
1344	else
1345		eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC;
1346
1347	rx_func_get(eth_dev);
1348
1349	/* Setting up rx_header size for the device */
1350	if (vtpci_with_feature(hw, VIRTIO_NET_F_MRG_RXBUF) ||
1351	    vtpci_with_feature(hw, VIRTIO_F_VERSION_1))
1352		hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr_mrg_rxbuf);
1353	else
1354		hw->vtnet_hdr_size = sizeof(struct virtio_net_hdr);
1355
1356	/* Copy the permanent MAC address to: virtio_hw */
1357	virtio_get_hwaddr(hw);
1358	ether_addr_copy((struct ether_addr *) hw->mac_addr,
1359			&eth_dev->data->mac_addrs[0]);
1360	PMD_INIT_LOG(DEBUG,
1361		     "PORT MAC: %02X:%02X:%02X:%02X:%02X:%02X",
1362		     hw->mac_addr[0], hw->mac_addr[1], hw->mac_addr[2],
1363		     hw->mac_addr[3], hw->mac_addr[4], hw->mac_addr[5]);
1364
1365	if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VQ)) {
1366		config = &local_config;
1367
1368		vtpci_read_dev_config(hw,
1369			offsetof(struct virtio_net_config, mac),
1370			&config->mac, sizeof(config->mac));
1371
1372		if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) {
1373			vtpci_read_dev_config(hw,
1374				offsetof(struct virtio_net_config, status),
1375				&config->status, sizeof(config->status));
1376		} else {
1377			PMD_INIT_LOG(DEBUG,
1378				     "VIRTIO_NET_F_STATUS is not supported");
1379			config->status = 0;
1380		}
1381
1382		if (vtpci_with_feature(hw, VIRTIO_NET_F_MQ)) {
1383			vtpci_read_dev_config(hw,
1384				offsetof(struct virtio_net_config, max_virtqueue_pairs),
1385				&config->max_virtqueue_pairs,
1386				sizeof(config->max_virtqueue_pairs));
1387		} else {
1388			PMD_INIT_LOG(DEBUG,
1389				     "VIRTIO_NET_F_MQ is not supported");
1390			config->max_virtqueue_pairs = 1;
1391		}
1392
1393		hw->max_queue_pairs = config->max_virtqueue_pairs;
1394
1395		PMD_INIT_LOG(DEBUG, "config->max_virtqueue_pairs=%d",
1396				config->max_virtqueue_pairs);
1397		PMD_INIT_LOG(DEBUG, "config->status=%d", config->status);
1398		PMD_INIT_LOG(DEBUG,
1399				"PORT MAC: %02X:%02X:%02X:%02X:%02X:%02X",
1400				config->mac[0], config->mac[1],
1401				config->mac[2], config->mac[3],
1402				config->mac[4], config->mac[5]);
1403	} else {
1404		PMD_INIT_LOG(DEBUG, "config->max_virtqueue_pairs=1");
1405		hw->max_queue_pairs = 1;
1406	}
1407
1408	ret = virtio_alloc_queues(eth_dev);
1409	if (ret < 0)
1410		return ret;
1411
1412	if (eth_dev->data->dev_conf.intr_conf.rxq) {
1413		if (virtio_configure_intr(eth_dev) < 0) {
1414			PMD_INIT_LOG(ERR, "failed to configure interrupt");
1415			return -1;
1416		}
1417	}
1418
1419	vtpci_reinit_complete(hw);
1420
1421	if (pci_dev)
1422		PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
1423			eth_dev->data->port_id, pci_dev->id.vendor_id,
1424			pci_dev->id.device_id);
1425
1426	return 0;
1427}
1428
1429/*
1430 * Remap the PCI device again (IO port map for legacy device and
1431 * memory map for modern device), so that the secondary process
1432 * could have the PCI initiated correctly.
1433 */
1434static int
1435virtio_remap_pci(struct rte_pci_device *pci_dev, struct virtio_hw *hw)
1436{
1437	if (hw->modern) {
1438		/*
1439		 * We don't have to re-parse the PCI config space, since
1440		 * rte_eal_pci_map_device() makes sure the mapped address
1441		 * in secondary process would equal to the one mapped in
1442		 * the primary process: error will be returned if that
1443		 * requirement is not met.
1444		 *
1445		 * That said, we could simply reuse all cap pointers
1446		 * (such as dev_cfg, common_cfg, etc.) parsed from the
1447		 * primary process, which is stored in shared memory.
1448		 */
1449		if (rte_eal_pci_map_device(pci_dev)) {
1450			PMD_INIT_LOG(DEBUG, "failed to map pci device!");
1451			return -1;
1452		}
1453	} else {
1454		if (rte_eal_pci_ioport_map(pci_dev, 0, VTPCI_IO(hw)) < 0)
1455			return -1;
1456	}
1457
1458	return 0;
1459}
1460
1461static void
1462virtio_set_vtpci_ops(struct virtio_hw *hw)
1463{
1464#ifdef RTE_VIRTIO_USER
1465	if (hw->virtio_user_dev)
1466		VTPCI_OPS(hw) = &virtio_user_ops;
1467	else
1468#endif
1469	if (hw->modern)
1470		VTPCI_OPS(hw) = &modern_ops;
1471	else
1472		VTPCI_OPS(hw) = &legacy_ops;
1473}
1474
1475/*
1476 * This function is based on probe() function in virtio_pci.c
1477 * It returns 0 on success.
1478 */
1479int
1480eth_virtio_dev_init(struct rte_eth_dev *eth_dev)
1481{
1482	struct virtio_hw *hw = eth_dev->data->dev_private;
1483	uint32_t dev_flags = RTE_ETH_DEV_DETACHABLE;
1484	int ret;
1485
1486	RTE_BUILD_BUG_ON(RTE_PKTMBUF_HEADROOM < sizeof(struct virtio_net_hdr_mrg_rxbuf));
1487
1488	eth_dev->dev_ops = &virtio_eth_dev_ops;
1489	eth_dev->tx_pkt_burst = &virtio_xmit_pkts;
1490
1491	if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
1492		if (!hw->virtio_user_dev) {
1493			ret = virtio_remap_pci(RTE_DEV_TO_PCI(eth_dev->device),
1494					       hw);
1495			if (ret)
1496				return ret;
1497		}
1498
1499		virtio_set_vtpci_ops(hw);
1500		if (hw->use_simple_rxtx) {
1501			eth_dev->tx_pkt_burst = virtio_xmit_pkts_simple;
1502			eth_dev->rx_pkt_burst = virtio_recv_pkts_vec;
1503		} else {
1504			rx_func_get(eth_dev);
1505		}
1506		return 0;
1507	}
1508
1509	/* Allocate memory for storing MAC addresses */
1510	eth_dev->data->mac_addrs = rte_zmalloc("virtio", VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN, 0);
1511	if (eth_dev->data->mac_addrs == NULL) {
1512		PMD_INIT_LOG(ERR,
1513			"Failed to allocate %d bytes needed to store MAC addresses",
1514			VIRTIO_MAX_MAC_ADDRS * ETHER_ADDR_LEN);
1515		return -ENOMEM;
1516	}
1517
1518	hw->port_id = eth_dev->data->port_id;
1519	/* For virtio_user case the hw->virtio_user_dev is populated by
1520	 * virtio_user_eth_dev_alloc() before eth_virtio_dev_init() is called.
1521	 */
1522	if (!hw->virtio_user_dev) {
1523		ret = vtpci_init(RTE_DEV_TO_PCI(eth_dev->device), hw,
1524				 &dev_flags);
1525		if (ret)
1526			return ret;
1527	}
1528
1529	eth_dev->data->dev_flags = dev_flags;
1530
1531	/* reset device and negotiate default features */
1532	ret = virtio_init_device(eth_dev, VIRTIO_PMD_DEFAULT_GUEST_FEATURES);
1533	if (ret < 0)
1534		return ret;
1535
1536	/* Setup interrupt callback  */
1537	if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
1538		rte_intr_callback_register(eth_dev->intr_handle,
1539			virtio_interrupt_handler, eth_dev);
1540
1541	return 0;
1542}
1543
1544static int
1545eth_virtio_dev_uninit(struct rte_eth_dev *eth_dev)
1546{
1547	PMD_INIT_FUNC_TRACE();
1548
1549	if (rte_eal_process_type() == RTE_PROC_SECONDARY)
1550		return -EPERM;
1551
1552	virtio_dev_stop(eth_dev);
1553	virtio_dev_close(eth_dev);
1554
1555	eth_dev->dev_ops = NULL;
1556	eth_dev->tx_pkt_burst = NULL;
1557	eth_dev->rx_pkt_burst = NULL;
1558
1559	rte_free(eth_dev->data->mac_addrs);
1560	eth_dev->data->mac_addrs = NULL;
1561
1562	/* reset interrupt callback  */
1563	if (eth_dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
1564		rte_intr_callback_unregister(eth_dev->intr_handle,
1565						virtio_interrupt_handler,
1566						eth_dev);
1567	if (eth_dev->device)
1568		rte_eal_pci_unmap_device(RTE_DEV_TO_PCI(eth_dev->device));
1569
1570	PMD_INIT_LOG(DEBUG, "dev_uninit completed");
1571
1572	return 0;
1573}
1574
1575static struct eth_driver rte_virtio_pmd = {
1576	.pci_drv = {
1577		.driver = {
1578			.name = "net_virtio",
1579		},
1580		.id_table = pci_id_virtio_map,
1581		.drv_flags = 0,
1582		.probe = rte_eth_dev_pci_probe,
1583		.remove = rte_eth_dev_pci_remove,
1584	},
1585	.eth_dev_init = eth_virtio_dev_init,
1586	.eth_dev_uninit = eth_virtio_dev_uninit,
1587	.dev_private_size = sizeof(struct virtio_hw),
1588};
1589
1590RTE_INIT(rte_virtio_pmd_init);
1591static void
1592rte_virtio_pmd_init(void)
1593{
1594	if (rte_eal_iopl_init() != 0) {
1595		PMD_INIT_LOG(ERR, "IOPL call failed - cannot use virtio PMD");
1596		return;
1597	}
1598
1599	rte_eal_pci_register(&rte_virtio_pmd.pci_drv);
1600}
1601
1602/*
1603 * Configure virtio device
1604 * It returns 0 on success.
1605 */
1606static int
1607virtio_dev_configure(struct rte_eth_dev *dev)
1608{
1609	const struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
1610	struct virtio_hw *hw = dev->data->dev_private;
1611	uint64_t req_features;
1612	int ret;
1613
1614	PMD_INIT_LOG(DEBUG, "configure");
1615	req_features = VIRTIO_PMD_DEFAULT_GUEST_FEATURES;
1616	if (rxmode->hw_ip_checksum)
1617		req_features |= (1ULL << VIRTIO_NET_F_GUEST_CSUM);
1618	if (rxmode->enable_lro)
1619		req_features |=
1620			(1ULL << VIRTIO_NET_F_GUEST_TSO4) |
1621			(1ULL << VIRTIO_NET_F_GUEST_TSO6);
1622
1623	/* if request features changed, reinit the device */
1624	if (req_features != hw->req_guest_features) {
1625		ret = virtio_init_device(dev, req_features);
1626		if (ret < 0)
1627			return ret;
1628	}
1629
1630	if (rxmode->hw_ip_checksum &&
1631		!vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_CSUM)) {
1632		PMD_DRV_LOG(NOTICE,
1633			"rx ip checksum not available on this host");
1634		return -ENOTSUP;
1635	}
1636
1637	if (rxmode->enable_lro &&
1638		(!vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4) ||
1639			!vtpci_with_feature(hw, VIRTIO_NET_F_GUEST_TSO4))) {
1640		PMD_DRV_LOG(NOTICE,
1641			"lro not available on this host");
1642		return -ENOTSUP;
1643	}
1644
1645	/* start control queue */
1646	if (vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VQ))
1647		virtio_dev_cq_start(dev);
1648
1649	hw->vlan_strip = rxmode->hw_vlan_strip;
1650
1651	if (rxmode->hw_vlan_filter
1652	    && !vtpci_with_feature(hw, VIRTIO_NET_F_CTRL_VLAN)) {
1653		PMD_DRV_LOG(NOTICE,
1654			    "vlan filtering not available on this host");
1655		return -ENOTSUP;
1656	}
1657
1658	if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
1659		/* Enable vector (0) for Link State Intrerrupt */
1660		if (VTPCI_OPS(hw)->set_config_irq(hw, 0) ==
1661				VIRTIO_MSI_NO_VECTOR) {
1662			PMD_DRV_LOG(ERR, "failed to set config vector");
1663			return -EBUSY;
1664		}
1665
1666	return 0;
1667}
1668
1669
1670static int
1671virtio_dev_start(struct rte_eth_dev *dev)
1672{
1673	uint16_t nb_queues, i;
1674	struct virtnet_rx *rxvq;
1675	struct virtnet_tx *txvq __rte_unused;
1676	struct virtio_hw *hw = dev->data->dev_private;
1677
1678	/* check if lsc interrupt feature is enabled */
1679	if (dev->data->dev_conf.intr_conf.lsc) {
1680		if (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)) {
1681			PMD_DRV_LOG(ERR, "link status not supported by host");
1682			return -ENOTSUP;
1683		}
1684	}
1685
1686	/* Enable uio/vfio intr/eventfd mapping: althrough we already did that
1687	 * in device configure, but it could be unmapped  when device is
1688	 * stopped.
1689	 */
1690	if (dev->data->dev_conf.intr_conf.lsc ||
1691	    dev->data->dev_conf.intr_conf.rxq) {
1692		rte_intr_disable(dev->intr_handle);
1693
1694		if (rte_intr_enable(dev->intr_handle) < 0) {
1695			PMD_DRV_LOG(ERR, "interrupt enable failed");
1696			return -EIO;
1697		}
1698	}
1699
1700	/* Initialize Link state */
1701	virtio_dev_link_update(dev, 0);
1702
1703	/*Notify the backend
1704	 *Otherwise the tap backend might already stop its queue due to fullness.
1705	 *vhost backend will have no chance to be waked up
1706	 */
1707	nb_queues = RTE_MAX(dev->data->nb_rx_queues, dev->data->nb_tx_queues);
1708	if (hw->max_queue_pairs > 1) {
1709		if (virtio_set_multiple_queues(dev, nb_queues) != 0)
1710			return -EINVAL;
1711	}
1712
1713	PMD_INIT_LOG(DEBUG, "nb_queues=%d", nb_queues);
1714
1715	for (i = 0; i < dev->data->nb_rx_queues; i++) {
1716		rxvq = dev->data->rx_queues[i];
1717		virtqueue_notify(rxvq->vq);
1718	}
1719
1720	PMD_INIT_LOG(DEBUG, "Notified backend at initialization");
1721
1722	for (i = 0; i < dev->data->nb_rx_queues; i++) {
1723		rxvq = dev->data->rx_queues[i];
1724		VIRTQUEUE_DUMP(rxvq->vq);
1725	}
1726
1727	for (i = 0; i < dev->data->nb_tx_queues; i++) {
1728		txvq = dev->data->tx_queues[i];
1729		VIRTQUEUE_DUMP(txvq->vq);
1730	}
1731
1732	return 0;
1733}
1734
1735static void virtio_dev_free_mbufs(struct rte_eth_dev *dev)
1736{
1737	struct rte_mbuf *buf;
1738	int i, mbuf_num = 0;
1739
1740	for (i = 0; i < dev->data->nb_rx_queues; i++) {
1741		struct virtnet_rx *rxvq = dev->data->rx_queues[i];
1742
1743		PMD_INIT_LOG(DEBUG,
1744			     "Before freeing rxq[%d] used and unused buf", i);
1745		VIRTQUEUE_DUMP(rxvq->vq);
1746
1747		PMD_INIT_LOG(DEBUG, "rx_queues[%d]=%p", i, rxvq);
1748		while ((buf = virtqueue_detatch_unused(rxvq->vq)) != NULL) {
1749			rte_pktmbuf_free(buf);
1750			mbuf_num++;
1751		}
1752
1753		PMD_INIT_LOG(DEBUG, "free %d mbufs", mbuf_num);
1754		PMD_INIT_LOG(DEBUG,
1755			     "After freeing rxq[%d] used and unused buf", i);
1756		VIRTQUEUE_DUMP(rxvq->vq);
1757	}
1758
1759	for (i = 0; i < dev->data->nb_tx_queues; i++) {
1760		struct virtnet_tx *txvq = dev->data->tx_queues[i];
1761
1762		PMD_INIT_LOG(DEBUG,
1763			     "Before freeing txq[%d] used and unused bufs",
1764			     i);
1765		VIRTQUEUE_DUMP(txvq->vq);
1766
1767		mbuf_num = 0;
1768		while ((buf = virtqueue_detatch_unused(txvq->vq)) != NULL) {
1769			rte_pktmbuf_free(buf);
1770			mbuf_num++;
1771		}
1772
1773		PMD_INIT_LOG(DEBUG, "free %d mbufs", mbuf_num);
1774		PMD_INIT_LOG(DEBUG,
1775			     "After freeing txq[%d] used and unused buf", i);
1776		VIRTQUEUE_DUMP(txvq->vq);
1777	}
1778}
1779
1780/*
1781 * Stop device: disable interrupt and mark link down
1782 */
1783static void
1784virtio_dev_stop(struct rte_eth_dev *dev)
1785{
1786	struct rte_eth_link link;
1787	struct rte_intr_conf *intr_conf = &dev->data->dev_conf.intr_conf;
1788
1789	PMD_INIT_LOG(DEBUG, "stop");
1790
1791	if (intr_conf->lsc || intr_conf->rxq)
1792		rte_intr_disable(dev->intr_handle);
1793
1794	memset(&link, 0, sizeof(link));
1795	virtio_dev_atomic_write_link_status(dev, &link);
1796}
1797
1798static int
1799virtio_dev_link_update(struct rte_eth_dev *dev, __rte_unused int wait_to_complete)
1800{
1801	struct rte_eth_link link, old;
1802	uint16_t status;
1803	struct virtio_hw *hw = dev->data->dev_private;
1804	memset(&link, 0, sizeof(link));
1805	virtio_dev_atomic_read_link_status(dev, &link);
1806	old = link;
1807	link.link_duplex = ETH_LINK_FULL_DUPLEX;
1808	link.link_speed  = SPEED_10G;
1809
1810	if (vtpci_with_feature(hw, VIRTIO_NET_F_STATUS)) {
1811		PMD_INIT_LOG(DEBUG, "Get link status from hw");
1812		vtpci_read_dev_config(hw,
1813				offsetof(struct virtio_net_config, status),
1814				&status, sizeof(status));
1815		if ((status & VIRTIO_NET_S_LINK_UP) == 0) {
1816			link.link_status = ETH_LINK_DOWN;
1817			PMD_INIT_LOG(DEBUG, "Port %d is down",
1818				     dev->data->port_id);
1819		} else {
1820			link.link_status = ETH_LINK_UP;
1821			PMD_INIT_LOG(DEBUG, "Port %d is up",
1822				     dev->data->port_id);
1823		}
1824	} else {
1825		link.link_status = ETH_LINK_UP;
1826	}
1827	virtio_dev_atomic_write_link_status(dev, &link);
1828
1829	return (old.link_status == link.link_status) ? -1 : 0;
1830}
1831
1832static void
1833virtio_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
1834{
1835	uint64_t tso_mask, host_features;
1836	struct virtio_hw *hw = dev->data->dev_private;
1837
1838	dev_info->speed_capa = ETH_LINK_SPEED_10G; /* fake value */
1839
1840	dev_info->pci_dev = dev->device ? RTE_DEV_TO_PCI(dev->device) : NULL;
1841	dev_info->max_rx_queues =
1842		RTE_MIN(hw->max_queue_pairs, VIRTIO_MAX_RX_QUEUES);
1843	dev_info->max_tx_queues =
1844		RTE_MIN(hw->max_queue_pairs, VIRTIO_MAX_TX_QUEUES);
1845	dev_info->min_rx_bufsize = VIRTIO_MIN_RX_BUFSIZE;
1846	dev_info->max_rx_pktlen = VIRTIO_MAX_RX_PKTLEN;
1847	dev_info->max_mac_addrs = VIRTIO_MAX_MAC_ADDRS;
1848	dev_info->default_txconf = (struct rte_eth_txconf) {
1849		.txq_flags = ETH_TXQ_FLAGS_NOOFFLOADS
1850	};
1851
1852	host_features = VTPCI_OPS(hw)->get_features(hw);
1853	dev_info->rx_offload_capa = 0;
1854	if (host_features & (1ULL << VIRTIO_NET_F_GUEST_CSUM)) {
1855		dev_info->rx_offload_capa |=
1856			DEV_RX_OFFLOAD_TCP_CKSUM |
1857			DEV_RX_OFFLOAD_UDP_CKSUM;
1858	}
1859	tso_mask = (1ULL << VIRTIO_NET_F_GUEST_TSO4) |
1860		(1ULL << VIRTIO_NET_F_GUEST_TSO6);
1861	if ((host_features & tso_mask) == tso_mask)
1862		dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TCP_LRO;
1863
1864	dev_info->tx_offload_capa = 0;
1865	if (hw->guest_features & (1ULL << VIRTIO_NET_F_CSUM)) {
1866		dev_info->tx_offload_capa |=
1867			DEV_TX_OFFLOAD_UDP_CKSUM |
1868			DEV_TX_OFFLOAD_TCP_CKSUM;
1869	}
1870	tso_mask = (1ULL << VIRTIO_NET_F_HOST_TSO4) |
1871		(1ULL << VIRTIO_NET_F_HOST_TSO6);
1872	if ((hw->guest_features & tso_mask) == tso_mask)
1873		dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO;
1874}
1875
1876/*
1877 * It enables testpmd to collect per queue stats.
1878 */
1879static int
1880virtio_dev_queue_stats_mapping_set(__rte_unused struct rte_eth_dev *eth_dev,
1881__rte_unused uint16_t queue_id, __rte_unused uint8_t stat_idx,
1882__rte_unused uint8_t is_rx)
1883{
1884	return 0;
1885}
1886
1887RTE_PMD_EXPORT_NAME(net_virtio, __COUNTER__);
1888RTE_PMD_REGISTER_PCI_TABLE(net_virtio, pci_id_virtio_map);
1889RTE_PMD_REGISTER_KMOD_DEP(net_virtio, "* igb_uio | uio_pci_generic | vfio");
1890