vhost_user.c revision 39157ec0
1/*-
2 *   BSD LICENSE
3 *
4 *   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
5 *   All rights reserved.
6 *
7 *   Redistribution and use in source and binary forms, with or without
8 *   modification, are permitted provided that the following conditions
9 *   are met:
10 *
11 *     * Redistributions of source code must retain the above copyright
12 *       notice, this list of conditions and the following disclaimer.
13 *     * Redistributions in binary form must reproduce the above copyright
14 *       notice, this list of conditions and the following disclaimer in
15 *       the documentation and/or other materials provided with the
16 *       distribution.
17 *     * Neither the name of Intel Corporation nor the names of its
18 *       contributors may be used to endorse or promote products derived
19 *       from this software without specific prior written permission.
20 *
21 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34#include <stdint.h>
35#include <stdio.h>
36#include <stdlib.h>
37#include <string.h>
38#include <unistd.h>
39#include <sys/mman.h>
40#include <sys/types.h>
41#include <sys/stat.h>
42#include <stdbool.h>
43#include <assert.h>
44#ifdef RTE_LIBRTE_VHOST_NUMA
45#include <numaif.h>
46#endif
47
48#include <rte_common.h>
49#include <rte_malloc.h>
50#include <rte_log.h>
51
52#include "vhost.h"
53#include "vhost_user.h"
54
55static const char *vhost_message_str[VHOST_USER_MAX] = {
56	[VHOST_USER_NONE] = "VHOST_USER_NONE",
57	[VHOST_USER_GET_FEATURES] = "VHOST_USER_GET_FEATURES",
58	[VHOST_USER_SET_FEATURES] = "VHOST_USER_SET_FEATURES",
59	[VHOST_USER_SET_OWNER] = "VHOST_USER_SET_OWNER",
60	[VHOST_USER_RESET_OWNER] = "VHOST_USER_RESET_OWNER",
61	[VHOST_USER_SET_MEM_TABLE] = "VHOST_USER_SET_MEM_TABLE",
62	[VHOST_USER_SET_LOG_BASE] = "VHOST_USER_SET_LOG_BASE",
63	[VHOST_USER_SET_LOG_FD] = "VHOST_USER_SET_LOG_FD",
64	[VHOST_USER_SET_VRING_NUM] = "VHOST_USER_SET_VRING_NUM",
65	[VHOST_USER_SET_VRING_ADDR] = "VHOST_USER_SET_VRING_ADDR",
66	[VHOST_USER_SET_VRING_BASE] = "VHOST_USER_SET_VRING_BASE",
67	[VHOST_USER_GET_VRING_BASE] = "VHOST_USER_GET_VRING_BASE",
68	[VHOST_USER_SET_VRING_KICK] = "VHOST_USER_SET_VRING_KICK",
69	[VHOST_USER_SET_VRING_CALL] = "VHOST_USER_SET_VRING_CALL",
70	[VHOST_USER_SET_VRING_ERR]  = "VHOST_USER_SET_VRING_ERR",
71	[VHOST_USER_GET_PROTOCOL_FEATURES]  = "VHOST_USER_GET_PROTOCOL_FEATURES",
72	[VHOST_USER_SET_PROTOCOL_FEATURES]  = "VHOST_USER_SET_PROTOCOL_FEATURES",
73	[VHOST_USER_GET_QUEUE_NUM]  = "VHOST_USER_GET_QUEUE_NUM",
74	[VHOST_USER_SET_VRING_ENABLE]  = "VHOST_USER_SET_VRING_ENABLE",
75	[VHOST_USER_SEND_RARP]  = "VHOST_USER_SEND_RARP",
76};
77
78static uint64_t
79get_blk_size(int fd)
80{
81	struct stat stat;
82	int ret;
83
84	ret = fstat(fd, &stat);
85	return ret == -1 ? (uint64_t)-1 : (uint64_t)stat.st_blksize;
86}
87
88static void
89free_mem_region(struct virtio_net *dev)
90{
91	uint32_t i;
92	struct virtio_memory_region *reg;
93
94	if (!dev || !dev->mem)
95		return;
96
97	for (i = 0; i < dev->mem->nregions; i++) {
98		reg = &dev->mem->regions[i];
99		if (reg->host_user_addr) {
100			munmap(reg->mmap_addr, reg->mmap_size);
101			close(reg->fd);
102		}
103	}
104}
105
106void
107vhost_backend_cleanup(struct virtio_net *dev)
108{
109	if (dev->mem) {
110		free_mem_region(dev);
111		rte_free(dev->mem);
112		dev->mem = NULL;
113	}
114
115	free(dev->guest_pages);
116	dev->guest_pages = NULL;
117
118	if (dev->log_addr) {
119		munmap((void *)(uintptr_t)dev->log_addr, dev->log_size);
120		dev->log_addr = 0;
121	}
122}
123
124/*
125 * This function just returns success at the moment unless
126 * the device hasn't been initialised.
127 */
128static int
129vhost_user_set_owner(void)
130{
131	return 0;
132}
133
134static int
135vhost_user_reset_owner(struct virtio_net *dev)
136{
137	if (dev->flags & VIRTIO_DEV_RUNNING) {
138		dev->flags &= ~VIRTIO_DEV_RUNNING;
139		notify_ops->destroy_device(dev->vid);
140	}
141
142	cleanup_device(dev, 0);
143	reset_device(dev);
144	return 0;
145}
146
147/*
148 * The features that we support are requested.
149 */
150static uint64_t
151vhost_user_get_features(void)
152{
153	return VHOST_FEATURES;
154}
155
156/*
157 * We receive the negotiated features supported by us and the virtio device.
158 */
159static int
160vhost_user_set_features(struct virtio_net *dev, uint64_t features)
161{
162	if (features & ~VHOST_FEATURES)
163		return -1;
164
165	dev->features = features;
166	if (dev->features &
167		((1 << VIRTIO_NET_F_MRG_RXBUF) | (1ULL << VIRTIO_F_VERSION_1))) {
168		dev->vhost_hlen = sizeof(struct virtio_net_hdr_mrg_rxbuf);
169	} else {
170		dev->vhost_hlen = sizeof(struct virtio_net_hdr);
171	}
172	LOG_DEBUG(VHOST_CONFIG,
173		"(%d) mergeable RX buffers %s, virtio 1 %s\n",
174		dev->vid,
175		(dev->features & (1 << VIRTIO_NET_F_MRG_RXBUF)) ? "on" : "off",
176		(dev->features & (1ULL << VIRTIO_F_VERSION_1)) ? "on" : "off");
177
178	return 0;
179}
180
181/*
182 * The virtio device sends us the size of the descriptor ring.
183 */
184static int
185vhost_user_set_vring_num(struct virtio_net *dev,
186			 struct vhost_vring_state *state)
187{
188	struct vhost_virtqueue *vq = dev->virtqueue[state->index];
189
190	vq->size = state->num;
191
192	if (dev->dequeue_zero_copy) {
193		vq->nr_zmbuf = 0;
194		vq->last_zmbuf_idx = 0;
195		vq->zmbuf_size = vq->size;
196		vq->zmbufs = rte_zmalloc(NULL, vq->zmbuf_size *
197					 sizeof(struct zcopy_mbuf), 0);
198		if (vq->zmbufs == NULL) {
199			RTE_LOG(WARNING, VHOST_CONFIG,
200				"failed to allocate mem for zero copy; "
201				"zero copy is force disabled\n");
202			dev->dequeue_zero_copy = 0;
203		}
204	}
205
206	vq->shadow_used_ring = rte_malloc(NULL,
207				vq->size * sizeof(struct vring_used_elem),
208				RTE_CACHE_LINE_SIZE);
209	if (!vq->shadow_used_ring) {
210		RTE_LOG(ERR, VHOST_CONFIG,
211			"failed to allocate memory for shadow used ring.\n");
212		return -1;
213	}
214
215	return 0;
216}
217
218/*
219 * Reallocate virtio_dev and vhost_virtqueue data structure to make them on the
220 * same numa node as the memory of vring descriptor.
221 */
222#ifdef RTE_LIBRTE_VHOST_NUMA
223static struct virtio_net*
224numa_realloc(struct virtio_net *dev, int index)
225{
226	int oldnode, newnode;
227	struct virtio_net *old_dev;
228	struct vhost_virtqueue *old_vq, *vq;
229	int ret;
230
231	/*
232	 * vq is allocated on pairs, we should try to do realloc
233	 * on first queue of one queue pair only.
234	 */
235	if (index % VIRTIO_QNUM != 0)
236		return dev;
237
238	old_dev = dev;
239	vq = old_vq = dev->virtqueue[index];
240
241	ret = get_mempolicy(&newnode, NULL, 0, old_vq->desc,
242			    MPOL_F_NODE | MPOL_F_ADDR);
243
244	/* check if we need to reallocate vq */
245	ret |= get_mempolicy(&oldnode, NULL, 0, old_vq,
246			     MPOL_F_NODE | MPOL_F_ADDR);
247	if (ret) {
248		RTE_LOG(ERR, VHOST_CONFIG,
249			"Unable to get vq numa information.\n");
250		return dev;
251	}
252	if (oldnode != newnode) {
253		RTE_LOG(INFO, VHOST_CONFIG,
254			"reallocate vq from %d to %d node\n", oldnode, newnode);
255		vq = rte_malloc_socket(NULL, sizeof(*vq) * VIRTIO_QNUM, 0,
256				       newnode);
257		if (!vq)
258			return dev;
259
260		memcpy(vq, old_vq, sizeof(*vq) * VIRTIO_QNUM);
261		rte_free(old_vq);
262	}
263
264	/* check if we need to reallocate dev */
265	ret = get_mempolicy(&oldnode, NULL, 0, old_dev,
266			    MPOL_F_NODE | MPOL_F_ADDR);
267	if (ret) {
268		RTE_LOG(ERR, VHOST_CONFIG,
269			"Unable to get dev numa information.\n");
270		goto out;
271	}
272	if (oldnode != newnode) {
273		RTE_LOG(INFO, VHOST_CONFIG,
274			"reallocate dev from %d to %d node\n",
275			oldnode, newnode);
276		dev = rte_malloc_socket(NULL, sizeof(*dev), 0, newnode);
277		if (!dev) {
278			dev = old_dev;
279			goto out;
280		}
281
282		memcpy(dev, old_dev, sizeof(*dev));
283		rte_free(old_dev);
284	}
285
286out:
287	dev->virtqueue[index] = vq;
288	dev->virtqueue[index + 1] = vq + 1;
289	vhost_devices[dev->vid] = dev;
290
291	return dev;
292}
293#else
294static struct virtio_net*
295numa_realloc(struct virtio_net *dev, int index __rte_unused)
296{
297	return dev;
298}
299#endif
300
301/*
302 * Converts QEMU virtual address to Vhost virtual address. This function is
303 * used to convert the ring addresses to our address space.
304 */
305static uint64_t
306qva_to_vva(struct virtio_net *dev, uint64_t qva)
307{
308	struct virtio_memory_region *reg;
309	uint32_t i;
310
311	/* Find the region where the address lives. */
312	for (i = 0; i < dev->mem->nregions; i++) {
313		reg = &dev->mem->regions[i];
314
315		if (qva >= reg->guest_user_addr &&
316		    qva <  reg->guest_user_addr + reg->size) {
317			return qva - reg->guest_user_addr +
318			       reg->host_user_addr;
319		}
320	}
321
322	return 0;
323}
324
325/*
326 * The virtio device sends us the desc, used and avail ring addresses.
327 * This function then converts these to our address space.
328 */
329static int
330vhost_user_set_vring_addr(struct virtio_net *dev, struct vhost_vring_addr *addr)
331{
332	struct vhost_virtqueue *vq;
333
334	if (dev->mem == NULL)
335		return -1;
336
337	/* addr->index refers to the queue index. The txq 1, rxq is 0. */
338	vq = dev->virtqueue[addr->index];
339
340	/* The addresses are converted from QEMU virtual to Vhost virtual. */
341	vq->desc = (struct vring_desc *)(uintptr_t)qva_to_vva(dev,
342			addr->desc_user_addr);
343	if (vq->desc == 0) {
344		RTE_LOG(ERR, VHOST_CONFIG,
345			"(%d) failed to find desc ring address.\n",
346			dev->vid);
347		return -1;
348	}
349
350	dev = numa_realloc(dev, addr->index);
351	vq = dev->virtqueue[addr->index];
352
353	vq->avail = (struct vring_avail *)(uintptr_t)qva_to_vva(dev,
354			addr->avail_user_addr);
355	if (vq->avail == 0) {
356		RTE_LOG(ERR, VHOST_CONFIG,
357			"(%d) failed to find avail ring address.\n",
358			dev->vid);
359		return -1;
360	}
361
362	vq->used = (struct vring_used *)(uintptr_t)qva_to_vva(dev,
363			addr->used_user_addr);
364	if (vq->used == 0) {
365		RTE_LOG(ERR, VHOST_CONFIG,
366			"(%d) failed to find used ring address.\n",
367			dev->vid);
368		return -1;
369	}
370
371	if (vq->last_used_idx != vq->used->idx) {
372		RTE_LOG(WARNING, VHOST_CONFIG,
373			"last_used_idx (%u) and vq->used->idx (%u) mismatches; "
374			"some packets maybe resent for Tx and dropped for Rx\n",
375			vq->last_used_idx, vq->used->idx);
376		vq->last_used_idx  = vq->used->idx;
377		vq->last_avail_idx = vq->used->idx;
378	}
379
380	vq->log_guest_addr = addr->log_guest_addr;
381
382	LOG_DEBUG(VHOST_CONFIG, "(%d) mapped address desc: %p\n",
383			dev->vid, vq->desc);
384	LOG_DEBUG(VHOST_CONFIG, "(%d) mapped address avail: %p\n",
385			dev->vid, vq->avail);
386	LOG_DEBUG(VHOST_CONFIG, "(%d) mapped address used: %p\n",
387			dev->vid, vq->used);
388	LOG_DEBUG(VHOST_CONFIG, "(%d) log_guest_addr: %" PRIx64 "\n",
389			dev->vid, vq->log_guest_addr);
390
391	return 0;
392}
393
394/*
395 * The virtio device sends us the available ring last used index.
396 */
397static int
398vhost_user_set_vring_base(struct virtio_net *dev,
399			  struct vhost_vring_state *state)
400{
401	dev->virtqueue[state->index]->last_used_idx  = state->num;
402	dev->virtqueue[state->index]->last_avail_idx = state->num;
403
404	return 0;
405}
406
407static void
408add_one_guest_page(struct virtio_net *dev, uint64_t guest_phys_addr,
409		   uint64_t host_phys_addr, uint64_t size)
410{
411	struct guest_page *page, *last_page;
412
413	if (dev->nr_guest_pages == dev->max_guest_pages) {
414		dev->max_guest_pages *= 2;
415		dev->guest_pages = realloc(dev->guest_pages,
416					dev->max_guest_pages * sizeof(*page));
417	}
418
419	if (dev->nr_guest_pages > 0) {
420		last_page = &dev->guest_pages[dev->nr_guest_pages - 1];
421		/* merge if the two pages are continuous */
422		if (host_phys_addr == last_page->host_phys_addr +
423				      last_page->size) {
424			last_page->size += size;
425			return;
426		}
427	}
428
429	page = &dev->guest_pages[dev->nr_guest_pages++];
430	page->guest_phys_addr = guest_phys_addr;
431	page->host_phys_addr  = host_phys_addr;
432	page->size = size;
433}
434
435static void
436add_guest_pages(struct virtio_net *dev, struct virtio_memory_region *reg,
437		uint64_t page_size)
438{
439	uint64_t reg_size = reg->size;
440	uint64_t host_user_addr  = reg->host_user_addr;
441	uint64_t guest_phys_addr = reg->guest_phys_addr;
442	uint64_t host_phys_addr;
443	uint64_t size;
444
445	host_phys_addr = rte_mem_virt2phy((void *)(uintptr_t)host_user_addr);
446	size = page_size - (guest_phys_addr & (page_size - 1));
447	size = RTE_MIN(size, reg_size);
448
449	add_one_guest_page(dev, guest_phys_addr, host_phys_addr, size);
450	host_user_addr  += size;
451	guest_phys_addr += size;
452	reg_size -= size;
453
454	while (reg_size > 0) {
455		size = RTE_MIN(reg_size, page_size);
456		host_phys_addr = rte_mem_virt2phy((void *)(uintptr_t)
457						  host_user_addr);
458		add_one_guest_page(dev, guest_phys_addr, host_phys_addr, size);
459
460		host_user_addr  += size;
461		guest_phys_addr += size;
462		reg_size -= size;
463	}
464}
465
466#ifdef RTE_LIBRTE_VHOST_DEBUG
467/* TODO: enable it only in debug mode? */
468static void
469dump_guest_pages(struct virtio_net *dev)
470{
471	uint32_t i;
472	struct guest_page *page;
473
474	for (i = 0; i < dev->nr_guest_pages; i++) {
475		page = &dev->guest_pages[i];
476
477		RTE_LOG(INFO, VHOST_CONFIG,
478			"guest physical page region %u\n"
479			"\t guest_phys_addr: %" PRIx64 "\n"
480			"\t host_phys_addr : %" PRIx64 "\n"
481			"\t size           : %" PRIx64 "\n",
482			i,
483			page->guest_phys_addr,
484			page->host_phys_addr,
485			page->size);
486	}
487}
488#else
489#define dump_guest_pages(dev)
490#endif
491
492static bool
493vhost_memory_changed(struct VhostUserMemory *new,
494		      struct virtio_memory *old)
495{
496	uint32_t i;
497
498	if (new->nregions != old->nregions)
499		return true;
500
501	for (i = 0; i < new->nregions; ++i) {
502		VhostUserMemoryRegion *new_r = &new->regions[i];
503		struct virtio_memory_region *old_r = &old->regions[i];
504
505		if (new_r->guest_phys_addr != old_r->guest_phys_addr)
506			return true;
507		if (new_r->memory_size != old_r->size)
508			return true;
509		if (new_r->userspace_addr != old_r->guest_user_addr)
510			return true;
511	}
512
513	return false;
514}
515
516static int
517vhost_user_set_mem_table(struct virtio_net *dev, struct VhostUserMsg *pmsg)
518{
519	struct VhostUserMemory memory = pmsg->payload.memory;
520	struct virtio_memory_region *reg;
521	void *mmap_addr;
522	uint64_t mmap_size;
523	uint64_t mmap_offset;
524	uint64_t alignment;
525	uint32_t i;
526	int fd;
527
528	if (dev->mem && !vhost_memory_changed(&memory, dev->mem)) {
529		RTE_LOG(INFO, VHOST_CONFIG,
530			"(%d) memory regions not changed\n", dev->vid);
531
532		for (i = 0; i < memory.nregions; i++)
533			close(pmsg->fds[i]);
534
535		return 0;
536	}
537
538	/* Remove from the data plane. */
539	if (dev->flags & VIRTIO_DEV_RUNNING) {
540		dev->flags &= ~VIRTIO_DEV_RUNNING;
541		notify_ops->destroy_device(dev->vid);
542	}
543
544	if (dev->mem) {
545		free_mem_region(dev);
546		rte_free(dev->mem);
547		dev->mem = NULL;
548	}
549
550	dev->nr_guest_pages = 0;
551	if (!dev->guest_pages) {
552		dev->max_guest_pages = 8;
553		dev->guest_pages = malloc(dev->max_guest_pages *
554						sizeof(struct guest_page));
555	}
556
557	dev->mem = rte_zmalloc("vhost-mem-table", sizeof(struct virtio_memory) +
558		sizeof(struct virtio_memory_region) * memory.nregions, 0);
559	if (dev->mem == NULL) {
560		RTE_LOG(ERR, VHOST_CONFIG,
561			"(%d) failed to allocate memory for dev->mem\n",
562			dev->vid);
563		return -1;
564	}
565	dev->mem->nregions = memory.nregions;
566
567	for (i = 0; i < memory.nregions; i++) {
568		fd  = pmsg->fds[i];
569		reg = &dev->mem->regions[i];
570
571		reg->guest_phys_addr = memory.regions[i].guest_phys_addr;
572		reg->guest_user_addr = memory.regions[i].userspace_addr;
573		reg->size            = memory.regions[i].memory_size;
574		reg->fd              = fd;
575
576		mmap_offset = memory.regions[i].mmap_offset;
577		mmap_size   = reg->size + mmap_offset;
578
579		/* mmap() without flag of MAP_ANONYMOUS, should be called
580		 * with length argument aligned with hugepagesz at older
581		 * longterm version Linux, like 2.6.32 and 3.2.72, or
582		 * mmap() will fail with EINVAL.
583		 *
584		 * to avoid failure, make sure in caller to keep length
585		 * aligned.
586		 */
587		alignment = get_blk_size(fd);
588		if (alignment == (uint64_t)-1) {
589			RTE_LOG(ERR, VHOST_CONFIG,
590				"couldn't get hugepage size through fstat\n");
591			goto err_mmap;
592		}
593		mmap_size = RTE_ALIGN_CEIL(mmap_size, alignment);
594
595		mmap_addr = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE,
596				 MAP_SHARED | MAP_POPULATE, fd, 0);
597
598		if (mmap_addr == MAP_FAILED) {
599			RTE_LOG(ERR, VHOST_CONFIG,
600				"mmap region %u failed.\n", i);
601			goto err_mmap;
602		}
603
604		reg->mmap_addr = mmap_addr;
605		reg->mmap_size = mmap_size;
606		reg->host_user_addr = (uint64_t)(uintptr_t)mmap_addr +
607				      mmap_offset;
608
609		if (dev->dequeue_zero_copy)
610			add_guest_pages(dev, reg, alignment);
611
612		RTE_LOG(INFO, VHOST_CONFIG,
613			"guest memory region %u, size: 0x%" PRIx64 "\n"
614			"\t guest physical addr: 0x%" PRIx64 "\n"
615			"\t guest virtual  addr: 0x%" PRIx64 "\n"
616			"\t host  virtual  addr: 0x%" PRIx64 "\n"
617			"\t mmap addr : 0x%" PRIx64 "\n"
618			"\t mmap size : 0x%" PRIx64 "\n"
619			"\t mmap align: 0x%" PRIx64 "\n"
620			"\t mmap off  : 0x%" PRIx64 "\n",
621			i, reg->size,
622			reg->guest_phys_addr,
623			reg->guest_user_addr,
624			reg->host_user_addr,
625			(uint64_t)(uintptr_t)mmap_addr,
626			mmap_size,
627			alignment,
628			mmap_offset);
629	}
630
631	dump_guest_pages(dev);
632
633	return 0;
634
635err_mmap:
636	free_mem_region(dev);
637	rte_free(dev->mem);
638	dev->mem = NULL;
639	return -1;
640}
641
642static int
643vq_is_ready(struct vhost_virtqueue *vq)
644{
645	return vq && vq->desc   &&
646	       vq->kickfd != VIRTIO_UNINITIALIZED_EVENTFD &&
647	       vq->callfd != VIRTIO_UNINITIALIZED_EVENTFD;
648}
649
650static int
651virtio_is_ready(struct virtio_net *dev)
652{
653	struct vhost_virtqueue *rvq, *tvq;
654	uint32_t i;
655
656	for (i = 0; i < dev->virt_qp_nb; i++) {
657		rvq = dev->virtqueue[i * VIRTIO_QNUM + VIRTIO_RXQ];
658		tvq = dev->virtqueue[i * VIRTIO_QNUM + VIRTIO_TXQ];
659
660		if (!vq_is_ready(rvq) || !vq_is_ready(tvq)) {
661			RTE_LOG(INFO, VHOST_CONFIG,
662				"virtio is not ready for processing.\n");
663			return 0;
664		}
665	}
666
667	RTE_LOG(INFO, VHOST_CONFIG,
668		"virtio is now ready for processing.\n");
669	return 1;
670}
671
672static void
673vhost_user_set_vring_call(struct virtio_net *dev, struct VhostUserMsg *pmsg)
674{
675	struct vhost_vring_file file;
676	struct vhost_virtqueue *vq;
677	uint32_t cur_qp_idx;
678
679	file.index = pmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
680	if (pmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK)
681		file.fd = VIRTIO_INVALID_EVENTFD;
682	else
683		file.fd = pmsg->fds[0];
684	RTE_LOG(INFO, VHOST_CONFIG,
685		"vring call idx:%d file:%d\n", file.index, file.fd);
686
687	/*
688	 * FIXME: VHOST_SET_VRING_CALL is the first per-vring message
689	 * we get, so we do vring queue pair allocation here.
690	 */
691	cur_qp_idx = file.index / VIRTIO_QNUM;
692	if (cur_qp_idx + 1 > dev->virt_qp_nb) {
693		if (alloc_vring_queue_pair(dev, cur_qp_idx) < 0)
694			return;
695	}
696
697	vq = dev->virtqueue[file.index];
698	assert(vq != NULL);
699
700	if (vq->callfd >= 0)
701		close(vq->callfd);
702
703	vq->callfd = file.fd;
704}
705
706/*
707 *  In vhost-user, when we receive kick message, will test whether virtio
708 *  device is ready for packet processing.
709 */
710static void
711vhost_user_set_vring_kick(struct virtio_net *dev, struct VhostUserMsg *pmsg)
712{
713	struct vhost_vring_file file;
714	struct vhost_virtqueue *vq;
715
716	file.index = pmsg->payload.u64 & VHOST_USER_VRING_IDX_MASK;
717	if (pmsg->payload.u64 & VHOST_USER_VRING_NOFD_MASK)
718		file.fd = VIRTIO_INVALID_EVENTFD;
719	else
720		file.fd = pmsg->fds[0];
721	RTE_LOG(INFO, VHOST_CONFIG,
722		"vring kick idx:%d file:%d\n", file.index, file.fd);
723
724	vq = dev->virtqueue[file.index];
725	if (vq->kickfd >= 0)
726		close(vq->kickfd);
727	vq->kickfd = file.fd;
728
729	if (virtio_is_ready(dev) && !(dev->flags & VIRTIO_DEV_RUNNING)) {
730		if (dev->dequeue_zero_copy) {
731			RTE_LOG(INFO, VHOST_CONFIG,
732				"dequeue zero copy is enabled\n");
733		}
734
735		if (notify_ops->new_device(dev->vid) == 0)
736			dev->flags |= VIRTIO_DEV_RUNNING;
737	}
738}
739
740static void
741free_zmbufs(struct vhost_virtqueue *vq)
742{
743	struct zcopy_mbuf *zmbuf, *next;
744
745	for (zmbuf = TAILQ_FIRST(&vq->zmbuf_list);
746	     zmbuf != NULL; zmbuf = next) {
747		next = TAILQ_NEXT(zmbuf, next);
748
749		rte_pktmbuf_free(zmbuf->mbuf);
750		TAILQ_REMOVE(&vq->zmbuf_list, zmbuf, next);
751	}
752
753	rte_free(vq->zmbufs);
754}
755
756/*
757 * when virtio is stopped, qemu will send us the GET_VRING_BASE message.
758 */
759static int
760vhost_user_get_vring_base(struct virtio_net *dev,
761			  struct vhost_vring_state *state)
762{
763	struct vhost_virtqueue *vq = dev->virtqueue[state->index];
764
765	/* We have to stop the queue (virtio) if it is running. */
766	if (dev->flags & VIRTIO_DEV_RUNNING) {
767		dev->flags &= ~VIRTIO_DEV_RUNNING;
768		notify_ops->destroy_device(dev->vid);
769	}
770
771	/* Here we are safe to get the last used index */
772	state->num = vq->last_used_idx;
773
774	RTE_LOG(INFO, VHOST_CONFIG,
775		"vring base idx:%d file:%d\n", state->index, state->num);
776	/*
777	 * Based on current qemu vhost-user implementation, this message is
778	 * sent and only sent in vhost_vring_stop.
779	 * TODO: cleanup the vring, it isn't usable since here.
780	 */
781	if (vq->kickfd >= 0)
782		close(vq->kickfd);
783
784	vq->kickfd = VIRTIO_UNINITIALIZED_EVENTFD;
785
786	if (dev->dequeue_zero_copy)
787		free_zmbufs(vq);
788	rte_free(vq->shadow_used_ring);
789	vq->shadow_used_ring = NULL;
790
791	return 0;
792}
793
794/*
795 * when virtio queues are ready to work, qemu will send us to
796 * enable the virtio queue pair.
797 */
798static int
799vhost_user_set_vring_enable(struct virtio_net *dev,
800			    struct vhost_vring_state *state)
801{
802	int enable = (int)state->num;
803
804	RTE_LOG(INFO, VHOST_CONFIG,
805		"set queue enable: %d to qp idx: %d\n",
806		enable, state->index);
807
808	if (notify_ops->vring_state_changed)
809		notify_ops->vring_state_changed(dev->vid, state->index, enable);
810
811	dev->virtqueue[state->index]->enabled = enable;
812
813	return 0;
814}
815
816static void
817vhost_user_set_protocol_features(struct virtio_net *dev,
818				 uint64_t protocol_features)
819{
820	if (protocol_features & ~VHOST_USER_PROTOCOL_FEATURES)
821		return;
822
823	dev->protocol_features = protocol_features;
824}
825
826static int
827vhost_user_set_log_base(struct virtio_net *dev, struct VhostUserMsg *msg)
828{
829	int fd = msg->fds[0];
830	uint64_t size, off;
831	void *addr;
832
833	if (fd < 0) {
834		RTE_LOG(ERR, VHOST_CONFIG, "invalid log fd: %d\n", fd);
835		return -1;
836	}
837
838	if (msg->size != sizeof(VhostUserLog)) {
839		RTE_LOG(ERR, VHOST_CONFIG,
840			"invalid log base msg size: %"PRId32" != %d\n",
841			msg->size, (int)sizeof(VhostUserLog));
842		return -1;
843	}
844
845	size = msg->payload.log.mmap_size;
846	off  = msg->payload.log.mmap_offset;
847	RTE_LOG(INFO, VHOST_CONFIG,
848		"log mmap size: %"PRId64", offset: %"PRId64"\n",
849		size, off);
850
851	/*
852	 * mmap from 0 to workaround a hugepage mmap bug: mmap will
853	 * fail when offset is not page size aligned.
854	 */
855	addr = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
856	close(fd);
857	if (addr == MAP_FAILED) {
858		RTE_LOG(ERR, VHOST_CONFIG, "mmap log base failed!\n");
859		return -1;
860	}
861
862	/*
863	 * Free previously mapped log memory on occasionally
864	 * multiple VHOST_USER_SET_LOG_BASE.
865	 */
866	if (dev->log_addr) {
867		munmap((void *)(uintptr_t)dev->log_addr, dev->log_size);
868	}
869	dev->log_addr = (uint64_t)(uintptr_t)addr;
870	dev->log_base = dev->log_addr + off;
871	dev->log_size = size;
872
873	return 0;
874}
875
876/*
877 * An rarp packet is constructed and broadcasted to notify switches about
878 * the new location of the migrated VM, so that packets from outside will
879 * not be lost after migration.
880 *
881 * However, we don't actually "send" a rarp packet here, instead, we set
882 * a flag 'broadcast_rarp' to let rte_vhost_dequeue_burst() inject it.
883 */
884static int
885vhost_user_send_rarp(struct virtio_net *dev, struct VhostUserMsg *msg)
886{
887	uint8_t *mac = (uint8_t *)&msg->payload.u64;
888
889	RTE_LOG(DEBUG, VHOST_CONFIG,
890		":: mac: %02x:%02x:%02x:%02x:%02x:%02x\n",
891		mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
892	memcpy(dev->mac.addr_bytes, mac, 6);
893
894	/*
895	 * Set the flag to inject a RARP broadcast packet at
896	 * rte_vhost_dequeue_burst().
897	 *
898	 * rte_smp_wmb() is for making sure the mac is copied
899	 * before the flag is set.
900	 */
901	rte_smp_wmb();
902	rte_atomic16_set(&dev->broadcast_rarp, 1);
903
904	return 0;
905}
906
907/* return bytes# of read on success or negative val on failure. */
908static int
909read_vhost_message(int sockfd, struct VhostUserMsg *msg)
910{
911	int ret;
912
913	ret = read_fd_message(sockfd, (char *)msg, VHOST_USER_HDR_SIZE,
914		msg->fds, VHOST_MEMORY_MAX_NREGIONS);
915	if (ret <= 0)
916		return ret;
917
918	if (msg && msg->size) {
919		if (msg->size > sizeof(msg->payload)) {
920			RTE_LOG(ERR, VHOST_CONFIG,
921				"invalid msg size: %d\n", msg->size);
922			return -1;
923		}
924		ret = read(sockfd, &msg->payload, msg->size);
925		if (ret <= 0)
926			return ret;
927		if (ret != (int)msg->size) {
928			RTE_LOG(ERR, VHOST_CONFIG,
929				"read control message failed\n");
930			return -1;
931		}
932	}
933
934	return ret;
935}
936
937static int
938send_vhost_message(int sockfd, struct VhostUserMsg *msg)
939{
940	int ret;
941
942	if (!msg)
943		return 0;
944
945	msg->flags &= ~VHOST_USER_VERSION_MASK;
946	msg->flags |= VHOST_USER_VERSION;
947	msg->flags |= VHOST_USER_REPLY_MASK;
948
949	ret = send_fd_message(sockfd, (char *)msg,
950		VHOST_USER_HDR_SIZE + msg->size, NULL, 0);
951
952	return ret;
953}
954
955static void
956vhost_user_lock_all_queue_pairs(struct virtio_net *dev)
957{
958	unsigned int i = 0;
959	unsigned int vq_num = 0;
960
961	while (vq_num < dev->virt_qp_nb * 2) {
962		struct vhost_virtqueue *vq = dev->virtqueue[i];
963
964		if (vq) {
965			rte_spinlock_lock(&vq->access_lock);
966			vq_num++;
967		}
968		i++;
969	}
970}
971
972static void
973vhost_user_unlock_all_queue_pairs(struct virtio_net *dev)
974{
975	unsigned int i = 0;
976	unsigned int vq_num = 0;
977
978	while (vq_num < dev->virt_qp_nb * 2) {
979		struct vhost_virtqueue *vq = dev->virtqueue[i];
980
981		if (vq) {
982			rte_spinlock_unlock(&vq->access_lock);
983			vq_num++;
984		}
985		i++;
986	}
987}
988
989int
990vhost_user_msg_handler(int vid, int fd)
991{
992	struct virtio_net *dev;
993	struct VhostUserMsg msg;
994	int ret;
995	int unlock_required = 0;
996
997	dev = get_device(vid);
998	if (dev == NULL)
999		return -1;
1000
1001	ret = read_vhost_message(fd, &msg);
1002	if (ret <= 0 || msg.request >= VHOST_USER_MAX) {
1003		if (ret < 0)
1004			RTE_LOG(ERR, VHOST_CONFIG,
1005				"vhost read message failed\n");
1006		else if (ret == 0)
1007			RTE_LOG(INFO, VHOST_CONFIG,
1008				"vhost peer closed\n");
1009		else
1010			RTE_LOG(ERR, VHOST_CONFIG,
1011				"vhost read incorrect message\n");
1012
1013		return -1;
1014	}
1015
1016	RTE_LOG(INFO, VHOST_CONFIG, "read message %s\n",
1017		vhost_message_str[msg.request]);
1018
1019	/*
1020	 * Note: we don't lock all queues on VHOST_USER_GET_VRING_BASE
1021	 * and VHOST_USER_RESET_OWNER, since it is sent when virtio stops
1022	 * and device is destroyed. destroy_device waits for queues to be
1023	 * inactive, so it is safe. Otherwise taking the access_lock
1024	 * would cause a dead lock.
1025	 */
1026	switch (msg.request) {
1027	case VHOST_USER_SET_FEATURES:
1028	case VHOST_USER_SET_PROTOCOL_FEATURES:
1029	case VHOST_USER_SET_OWNER:
1030	case VHOST_USER_SET_MEM_TABLE:
1031	case VHOST_USER_SET_LOG_BASE:
1032	case VHOST_USER_SET_LOG_FD:
1033	case VHOST_USER_SET_VRING_NUM:
1034	case VHOST_USER_SET_VRING_ADDR:
1035	case VHOST_USER_SET_VRING_BASE:
1036	case VHOST_USER_SET_VRING_KICK:
1037	case VHOST_USER_SET_VRING_CALL:
1038	case VHOST_USER_SET_VRING_ERR:
1039	case VHOST_USER_SET_VRING_ENABLE:
1040	case VHOST_USER_SEND_RARP:
1041		vhost_user_lock_all_queue_pairs(dev);
1042		unlock_required = 1;
1043		break;
1044	default:
1045		break;
1046
1047	}
1048
1049	switch (msg.request) {
1050	case VHOST_USER_GET_FEATURES:
1051		msg.payload.u64 = vhost_user_get_features();
1052		msg.size = sizeof(msg.payload.u64);
1053		send_vhost_message(fd, &msg);
1054		break;
1055	case VHOST_USER_SET_FEATURES:
1056		vhost_user_set_features(dev, msg.payload.u64);
1057		break;
1058
1059	case VHOST_USER_GET_PROTOCOL_FEATURES:
1060		msg.payload.u64 = VHOST_USER_PROTOCOL_FEATURES;
1061		msg.size = sizeof(msg.payload.u64);
1062		send_vhost_message(fd, &msg);
1063		break;
1064	case VHOST_USER_SET_PROTOCOL_FEATURES:
1065		vhost_user_set_protocol_features(dev, msg.payload.u64);
1066		break;
1067
1068	case VHOST_USER_SET_OWNER:
1069		vhost_user_set_owner();
1070		break;
1071	case VHOST_USER_RESET_OWNER:
1072		vhost_user_reset_owner(dev);
1073		break;
1074
1075	case VHOST_USER_SET_MEM_TABLE:
1076		vhost_user_set_mem_table(dev, &msg);
1077		break;
1078
1079	case VHOST_USER_SET_LOG_BASE:
1080		vhost_user_set_log_base(dev, &msg);
1081
1082		/* it needs a reply */
1083		msg.size = sizeof(msg.payload.u64);
1084		send_vhost_message(fd, &msg);
1085		break;
1086	case VHOST_USER_SET_LOG_FD:
1087		close(msg.fds[0]);
1088		RTE_LOG(INFO, VHOST_CONFIG, "not implemented.\n");
1089		break;
1090
1091	case VHOST_USER_SET_VRING_NUM:
1092		vhost_user_set_vring_num(dev, &msg.payload.state);
1093		break;
1094	case VHOST_USER_SET_VRING_ADDR:
1095		vhost_user_set_vring_addr(dev, &msg.payload.addr);
1096		break;
1097	case VHOST_USER_SET_VRING_BASE:
1098		vhost_user_set_vring_base(dev, &msg.payload.state);
1099		break;
1100
1101	case VHOST_USER_GET_VRING_BASE:
1102		ret = vhost_user_get_vring_base(dev, &msg.payload.state);
1103		msg.size = sizeof(msg.payload.state);
1104		send_vhost_message(fd, &msg);
1105		break;
1106
1107	case VHOST_USER_SET_VRING_KICK:
1108		vhost_user_set_vring_kick(dev, &msg);
1109		break;
1110	case VHOST_USER_SET_VRING_CALL:
1111		vhost_user_set_vring_call(dev, &msg);
1112		break;
1113
1114	case VHOST_USER_SET_VRING_ERR:
1115		if (!(msg.payload.u64 & VHOST_USER_VRING_NOFD_MASK))
1116			close(msg.fds[0]);
1117		RTE_LOG(INFO, VHOST_CONFIG, "not implemented\n");
1118		break;
1119
1120	case VHOST_USER_GET_QUEUE_NUM:
1121		msg.payload.u64 = VHOST_MAX_QUEUE_PAIRS;
1122		msg.size = sizeof(msg.payload.u64);
1123		send_vhost_message(fd, &msg);
1124		break;
1125
1126	case VHOST_USER_SET_VRING_ENABLE:
1127		vhost_user_set_vring_enable(dev, &msg.payload.state);
1128		break;
1129	case VHOST_USER_SEND_RARP:
1130		vhost_user_send_rarp(dev, &msg);
1131		break;
1132
1133	default:
1134		break;
1135
1136	}
1137
1138	if (unlock_required)
1139		vhost_user_unlock_all_queue_pairs(dev);
1140
1141	return 0;
1142}
1143