virtqueue.h revision 6b3e017e
1/*-
2 *   BSD LICENSE
3 *
4 *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
5 *   All rights reserved.
6 *
7 *   Redistribution and use in source and binary forms, with or without
8 *   modification, are permitted provided that the following conditions
9 *   are met:
10 *
11 *     * Redistributions of source code must retain the above copyright
12 *       notice, this list of conditions and the following disclaimer.
13 *     * Redistributions in binary form must reproduce the above copyright
14 *       notice, this list of conditions and the following disclaimer in
15 *       the documentation and/or other materials provided with the
16 *       distribution.
17 *     * Neither the name of Intel Corporation nor the names of its
18 *       contributors may be used to endorse or promote products derived
19 *       from this software without specific prior written permission.
20 *
21 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34#ifndef _VIRTQUEUE_H_
35#define _VIRTQUEUE_H_
36
37#include <stdint.h>
38
39#include <rte_atomic.h>
40#include <rte_memory.h>
41#include <rte_memzone.h>
42#include <rte_mempool.h>
43
44#include "virtio_pci.h"
45#include "virtio_ring.h"
46#include "virtio_logs.h"
47#include "virtio_rxtx.h"
48
49struct rte_mbuf;
50
51/*
52 * Per virtio_config.h in Linux.
53 *     For virtio_pci on SMP, we don't need to order with respect to MMIO
54 *     accesses through relaxed memory I/O windows, so smp_mb() et al are
55 *     sufficient.
56 *
57 */
58#define virtio_mb()	rte_smp_mb()
59#define virtio_rmb()	rte_smp_rmb()
60#define virtio_wmb()	rte_smp_wmb()
61
62#ifdef RTE_PMD_PACKET_PREFETCH
63#define rte_packet_prefetch(p)  rte_prefetch1(p)
64#else
65#define rte_packet_prefetch(p)  do {} while(0)
66#endif
67
68#define VIRTQUEUE_MAX_NAME_SZ 32
69
70#ifdef RTE_VIRTIO_USER
71/**
72 * Return the physical address (or virtual address in case of
73 * virtio-user) of mbuf data buffer.
74 */
75#define VIRTIO_MBUF_ADDR(mb, vq) (*(uint64_t *)((uintptr_t)(mb) + (vq)->offset))
76#else
77#define VIRTIO_MBUF_ADDR(mb, vq) ((mb)->buf_physaddr)
78#endif
79
80/**
81 * Return the physical address (or virtual address in case of
82 * virtio-user) of mbuf data buffer, taking care of mbuf data offset
83 */
84#define VIRTIO_MBUF_DATA_DMA_ADDR(mb, vq) \
85	(VIRTIO_MBUF_ADDR(mb, vq) + (mb)->data_off)
86
87#define VTNET_SQ_RQ_QUEUE_IDX 0
88#define VTNET_SQ_TQ_QUEUE_IDX 1
89#define VTNET_SQ_CQ_QUEUE_IDX 2
90
91enum { VTNET_RQ = 0, VTNET_TQ = 1, VTNET_CQ = 2 };
92/**
93 * The maximum virtqueue size is 2^15. Use that value as the end of
94 * descriptor chain terminator since it will never be a valid index
95 * in the descriptor table. This is used to verify we are correctly
96 * handling vq_free_cnt.
97 */
98#define VQ_RING_DESC_CHAIN_END 32768
99
100/**
101 * Control the RX mode, ie. promiscuous, allmulti, etc...
102 * All commands require an "out" sg entry containing a 1 byte
103 * state value, zero = disable, non-zero = enable.  Commands
104 * 0 and 1 are supported with the VIRTIO_NET_F_CTRL_RX feature.
105 * Commands 2-5 are added with VIRTIO_NET_F_CTRL_RX_EXTRA.
106 */
107#define VIRTIO_NET_CTRL_RX              0
108#define VIRTIO_NET_CTRL_RX_PROMISC      0
109#define VIRTIO_NET_CTRL_RX_ALLMULTI     1
110#define VIRTIO_NET_CTRL_RX_ALLUNI       2
111#define VIRTIO_NET_CTRL_RX_NOMULTI      3
112#define VIRTIO_NET_CTRL_RX_NOUNI        4
113#define VIRTIO_NET_CTRL_RX_NOBCAST      5
114
115/**
116 * Control the MAC
117 *
118 * The MAC filter table is managed by the hypervisor, the guest should
119 * assume the size is infinite.  Filtering should be considered
120 * non-perfect, ie. based on hypervisor resources, the guest may
121 * received packets from sources not specified in the filter list.
122 *
123 * In addition to the class/cmd header, the TABLE_SET command requires
124 * two out scatterlists.  Each contains a 4 byte count of entries followed
125 * by a concatenated byte stream of the ETH_ALEN MAC addresses.  The
126 * first sg list contains unicast addresses, the second is for multicast.
127 * This functionality is present if the VIRTIO_NET_F_CTRL_RX feature
128 * is available.
129 *
130 * The ADDR_SET command requests one out scatterlist, it contains a
131 * 6 bytes MAC address. This functionality is present if the
132 * VIRTIO_NET_F_CTRL_MAC_ADDR feature is available.
133 */
134struct virtio_net_ctrl_mac {
135	uint32_t entries;
136	uint8_t macs[][ETHER_ADDR_LEN];
137} __attribute__((__packed__));
138
139#define VIRTIO_NET_CTRL_MAC    1
140 #define VIRTIO_NET_CTRL_MAC_TABLE_SET        0
141 #define VIRTIO_NET_CTRL_MAC_ADDR_SET         1
142
143/**
144 * Control VLAN filtering
145 *
146 * The VLAN filter table is controlled via a simple ADD/DEL interface.
147 * VLAN IDs not added may be filtered by the hypervisor.  Del is the
148 * opposite of add.  Both commands expect an out entry containing a 2
149 * byte VLAN ID.  VLAN filtering is available with the
150 * VIRTIO_NET_F_CTRL_VLAN feature bit.
151 */
152#define VIRTIO_NET_CTRL_VLAN     2
153#define VIRTIO_NET_CTRL_VLAN_ADD 0
154#define VIRTIO_NET_CTRL_VLAN_DEL 1
155
156struct virtio_net_ctrl_hdr {
157	uint8_t class;
158	uint8_t cmd;
159} __attribute__((packed));
160
161typedef uint8_t virtio_net_ctrl_ack;
162
163#define VIRTIO_NET_OK     0
164#define VIRTIO_NET_ERR    1
165
166#define VIRTIO_MAX_CTRL_DATA 2048
167
168struct virtio_pmd_ctrl {
169	struct virtio_net_ctrl_hdr hdr;
170	virtio_net_ctrl_ack status;
171	uint8_t data[VIRTIO_MAX_CTRL_DATA];
172};
173
174struct vq_desc_extra {
175	void *cookie;
176	uint16_t ndescs;
177};
178
179struct virtqueue {
180	struct virtio_hw  *hw; /**< virtio_hw structure pointer. */
181	struct vring vq_ring;  /**< vring keeping desc, used and avail */
182	/**
183	 * Last consumed descriptor in the used table,
184	 * trails vq_ring.used->idx.
185	 */
186	uint16_t vq_used_cons_idx;
187	uint16_t vq_nentries;  /**< vring desc numbers */
188	uint16_t vq_free_cnt;  /**< num of desc available */
189	uint16_t vq_avail_idx; /**< sync until needed */
190	uint16_t vq_free_thresh; /**< free threshold */
191
192	void *vq_ring_virt_mem;  /**< linear address of vring*/
193	unsigned int vq_ring_size;
194
195	union {
196		struct virtnet_rx rxq;
197		struct virtnet_tx txq;
198		struct virtnet_ctl cq;
199	};
200
201	phys_addr_t vq_ring_mem; /**< physical address of vring,
202				  * or virtual address for virtio_user. */
203
204	/**
205	 * Head of the free chain in the descriptor table. If
206	 * there are no free descriptors, this will be set to
207	 * VQ_RING_DESC_CHAIN_END.
208	 */
209	uint16_t  vq_desc_head_idx;
210	uint16_t  vq_desc_tail_idx;
211	uint16_t  vq_queue_index;   /**< PCI queue index */
212	uint16_t offset; /**< relative offset to obtain addr in mbuf */
213	uint16_t  *notify_addr;
214	struct rte_mbuf **sw_ring;  /**< RX software ring. */
215	struct vq_desc_extra vq_descx[0];
216};
217
218/* If multiqueue is provided by host, then we suppport it. */
219#define VIRTIO_NET_CTRL_MQ   4
220#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET        0
221#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN        1
222#define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX        0x8000
223
224#define VIRTIO_NET_CTRL_MAC_ADDR_SET         1
225
226/**
227 * This is the first element of the scatter-gather list.  If you don't
228 * specify GSO or CSUM features, you can simply ignore the header.
229 */
230struct virtio_net_hdr {
231#define VIRTIO_NET_HDR_F_NEEDS_CSUM 1    /**< Use csum_start,csum_offset*/
232#define VIRTIO_NET_HDR_F_DATA_VALID 2    /**< Checksum is valid */
233	uint8_t flags;
234#define VIRTIO_NET_HDR_GSO_NONE     0    /**< Not a GSO frame */
235#define VIRTIO_NET_HDR_GSO_TCPV4    1    /**< GSO frame, IPv4 TCP (TSO) */
236#define VIRTIO_NET_HDR_GSO_UDP      3    /**< GSO frame, IPv4 UDP (UFO) */
237#define VIRTIO_NET_HDR_GSO_TCPV6    4    /**< GSO frame, IPv6 TCP */
238#define VIRTIO_NET_HDR_GSO_ECN      0x80 /**< TCP has ECN set */
239	uint8_t gso_type;
240	uint16_t hdr_len;     /**< Ethernet + IP + tcp/udp hdrs */
241	uint16_t gso_size;    /**< Bytes to append to hdr_len per frame */
242	uint16_t csum_start;  /**< Position to start checksumming from */
243	uint16_t csum_offset; /**< Offset after that to place checksum */
244};
245
246/**
247 * This is the version of the header to use when the MRG_RXBUF
248 * feature has been negotiated.
249 */
250struct virtio_net_hdr_mrg_rxbuf {
251	struct   virtio_net_hdr hdr;
252	uint16_t num_buffers; /**< Number of merged rx buffers */
253};
254
255/* Region reserved to allow for transmit header and indirect ring */
256#define VIRTIO_MAX_TX_INDIRECT 8
257struct virtio_tx_region {
258	struct virtio_net_hdr_mrg_rxbuf tx_hdr;
259	struct vring_desc tx_indir[VIRTIO_MAX_TX_INDIRECT]
260			   __attribute__((__aligned__(16)));
261};
262
263/* Chain all the descriptors in the ring with an END */
264static inline void
265vring_desc_init(struct vring_desc *dp, uint16_t n)
266{
267	uint16_t i;
268
269	for (i = 0; i < n - 1; i++)
270		dp[i].next = (uint16_t)(i + 1);
271	dp[i].next = VQ_RING_DESC_CHAIN_END;
272}
273
274/**
275 * Tell the backend not to interrupt us.
276 */
277void virtqueue_disable_intr(struct virtqueue *vq);
278/**
279 *  Dump virtqueue internal structures, for debug purpose only.
280 */
281void virtqueue_dump(struct virtqueue *vq);
282/**
283 *  Get all mbufs to be freed.
284 */
285struct rte_mbuf *virtqueue_detatch_unused(struct virtqueue *vq);
286
287static inline int
288virtqueue_full(const struct virtqueue *vq)
289{
290	return vq->vq_free_cnt == 0;
291}
292
293#define VIRTQUEUE_NUSED(vq) ((uint16_t)((vq)->vq_ring.used->idx - (vq)->vq_used_cons_idx))
294
295static inline void
296vq_update_avail_idx(struct virtqueue *vq)
297{
298	virtio_wmb();
299	vq->vq_ring.avail->idx = vq->vq_avail_idx;
300}
301
302static inline void
303vq_update_avail_ring(struct virtqueue *vq, uint16_t desc_idx)
304{
305	uint16_t avail_idx;
306	/*
307	 * Place the head of the descriptor chain into the next slot and make
308	 * it usable to the host. The chain is made available now rather than
309	 * deferring to virtqueue_notify() in the hopes that if the host is
310	 * currently running on another CPU, we can keep it processing the new
311	 * descriptor.
312	 */
313	avail_idx = (uint16_t)(vq->vq_avail_idx & (vq->vq_nentries - 1));
314	if (unlikely(vq->vq_ring.avail->ring[avail_idx] != desc_idx))
315		vq->vq_ring.avail->ring[avail_idx] = desc_idx;
316	vq->vq_avail_idx++;
317}
318
319static inline int
320virtqueue_kick_prepare(struct virtqueue *vq)
321{
322	return !(vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY);
323}
324
325static inline void
326virtqueue_notify(struct virtqueue *vq)
327{
328	/*
329	 * Ensure updated avail->idx is visible to host.
330	 * For virtio on IA, the notificaiton is through io port operation
331	 * which is a serialization instruction itself.
332	 */
333	vq->hw->vtpci_ops->notify_queue(vq->hw, vq);
334}
335
336#ifdef RTE_LIBRTE_VIRTIO_DEBUG_DUMP
337#define VIRTQUEUE_DUMP(vq) do { \
338	uint16_t used_idx, nused; \
339	used_idx = (vq)->vq_ring.used->idx; \
340	nused = (uint16_t)(used_idx - (vq)->vq_used_cons_idx); \
341	PMD_INIT_LOG(DEBUG, \
342	  "VQ: - size=%d; free=%d; used=%d; desc_head_idx=%d;" \
343	  " avail.idx=%d; used_cons_idx=%d; used.idx=%d;" \
344	  " avail.flags=0x%x; used.flags=0x%x", \
345	  (vq)->vq_nentries, (vq)->vq_free_cnt, nused, \
346	  (vq)->vq_desc_head_idx, (vq)->vq_ring.avail->idx, \
347	  (vq)->vq_used_cons_idx, (vq)->vq_ring.used->idx, \
348	  (vq)->vq_ring.avail->flags, (vq)->vq_ring.used->flags); \
349} while (0)
350#else
351#define VIRTQUEUE_DUMP(vq) do { } while (0)
352#endif
353
354#endif /* _VIRTQUEUE_H_ */
355