i40e_ethdev_vf.c revision c300e355
1/*-
2 *   BSD LICENSE
3 *
4 *   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
5 *   All rights reserved.
6 *
7 *   Redistribution and use in source and binary forms, with or without
8 *   modification, are permitted provided that the following conditions
9 *   are met:
10 *
11 *     * Redistributions of source code must retain the above copyright
12 *       notice, this list of conditions and the following disclaimer.
13 *     * Redistributions in binary form must reproduce the above copyright
14 *       notice, this list of conditions and the following disclaimer in
15 *       the documentation and/or other materials provided with the
16 *       distribution.
17 *     * Neither the name of Intel Corporation nor the names of its
18 *       contributors may be used to endorse or promote products derived
19 *       from this software without specific prior written permission.
20 *
21 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34#include <sys/queue.h>
35#include <stdio.h>
36#include <errno.h>
37#include <stdint.h>
38#include <string.h>
39#include <unistd.h>
40#include <stdarg.h>
41#include <inttypes.h>
42#include <rte_byteorder.h>
43#include <rte_common.h>
44#include <rte_cycles.h>
45
46#include <rte_interrupts.h>
47#include <rte_log.h>
48#include <rte_debug.h>
49#include <rte_pci.h>
50#include <rte_atomic.h>
51#include <rte_branch_prediction.h>
52#include <rte_memory.h>
53#include <rte_memzone.h>
54#include <rte_eal.h>
55#include <rte_alarm.h>
56#include <rte_ether.h>
57#include <rte_ethdev.h>
58#include <rte_atomic.h>
59#include <rte_malloc.h>
60#include <rte_dev.h>
61
62#include "i40e_logs.h"
63#include "base/i40e_prototype.h"
64#include "base/i40e_adminq_cmd.h"
65#include "base/i40e_type.h"
66
67#include "i40e_rxtx.h"
68#include "i40e_ethdev.h"
69#include "i40e_pf.h"
70#define I40EVF_VSI_DEFAULT_MSIX_INTR     1
71#define I40EVF_VSI_DEFAULT_MSIX_INTR_LNX 0
72
73/* busy wait delay in msec */
74#define I40EVF_BUSY_WAIT_DELAY 10
75#define I40EVF_BUSY_WAIT_COUNT 50
76#define MAX_RESET_WAIT_CNT     20
77
78struct i40evf_arq_msg_info {
79	enum i40e_virtchnl_ops ops;
80	enum i40e_status_code result;
81	uint16_t buf_len;
82	uint16_t msg_len;
83	uint8_t *msg;
84};
85
86struct vf_cmd_info {
87	enum i40e_virtchnl_ops ops;
88	uint8_t *in_args;
89	uint32_t in_args_size;
90	uint8_t *out_buffer;
91	/* Input & output type. pass in buffer size and pass out
92	 * actual return result
93	 */
94	uint32_t out_size;
95};
96
97enum i40evf_aq_result {
98	I40EVF_MSG_ERR = -1, /* Meet error when accessing admin queue */
99	I40EVF_MSG_NON,      /* Read nothing from admin queue */
100	I40EVF_MSG_SYS,      /* Read system msg from admin queue */
101	I40EVF_MSG_CMD,      /* Read async command result */
102};
103
104static int i40evf_dev_configure(struct rte_eth_dev *dev);
105static int i40evf_dev_start(struct rte_eth_dev *dev);
106static void i40evf_dev_stop(struct rte_eth_dev *dev);
107static void i40evf_dev_info_get(struct rte_eth_dev *dev,
108				struct rte_eth_dev_info *dev_info);
109static int i40evf_dev_link_update(struct rte_eth_dev *dev,
110				  __rte_unused int wait_to_complete);
111static void i40evf_dev_stats_get(struct rte_eth_dev *dev,
112				struct rte_eth_stats *stats);
113static int i40evf_dev_xstats_get(struct rte_eth_dev *dev,
114				 struct rte_eth_xstat *xstats, unsigned n);
115static int i40evf_dev_xstats_get_names(struct rte_eth_dev *dev,
116				       struct rte_eth_xstat_name *xstats_names,
117				       unsigned limit);
118static void i40evf_dev_xstats_reset(struct rte_eth_dev *dev);
119static int i40evf_vlan_filter_set(struct rte_eth_dev *dev,
120				  uint16_t vlan_id, int on);
121static void i40evf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
122static int i40evf_vlan_pvid_set(struct rte_eth_dev *dev, uint16_t pvid,
123				int on);
124static void i40evf_dev_close(struct rte_eth_dev *dev);
125static void i40evf_dev_promiscuous_enable(struct rte_eth_dev *dev);
126static void i40evf_dev_promiscuous_disable(struct rte_eth_dev *dev);
127static void i40evf_dev_allmulticast_enable(struct rte_eth_dev *dev);
128static void i40evf_dev_allmulticast_disable(struct rte_eth_dev *dev);
129static int i40evf_init_vlan(struct rte_eth_dev *dev);
130static int i40evf_dev_rx_queue_start(struct rte_eth_dev *dev,
131				     uint16_t rx_queue_id);
132static int i40evf_dev_rx_queue_stop(struct rte_eth_dev *dev,
133				    uint16_t rx_queue_id);
134static int i40evf_dev_tx_queue_start(struct rte_eth_dev *dev,
135				     uint16_t tx_queue_id);
136static int i40evf_dev_tx_queue_stop(struct rte_eth_dev *dev,
137				    uint16_t tx_queue_id);
138static void i40evf_add_mac_addr(struct rte_eth_dev *dev,
139				struct ether_addr *addr,
140				uint32_t index,
141				uint32_t pool);
142static void i40evf_del_mac_addr(struct rte_eth_dev *dev, uint32_t index);
143static int i40evf_dev_rss_reta_update(struct rte_eth_dev *dev,
144			struct rte_eth_rss_reta_entry64 *reta_conf,
145			uint16_t reta_size);
146static int i40evf_dev_rss_reta_query(struct rte_eth_dev *dev,
147			struct rte_eth_rss_reta_entry64 *reta_conf,
148			uint16_t reta_size);
149static int i40evf_config_rss(struct i40e_vf *vf);
150static int i40evf_dev_rss_hash_update(struct rte_eth_dev *dev,
151				      struct rte_eth_rss_conf *rss_conf);
152static int i40evf_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
153					struct rte_eth_rss_conf *rss_conf);
154static int
155i40evf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id);
156static int
157i40evf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id);
158static void i40evf_handle_pf_event(__rte_unused struct rte_eth_dev *dev,
159				   uint8_t *msg,
160				   uint16_t msglen);
161
162/* Default hash key buffer for RSS */
163static uint32_t rss_key_default[I40E_VFQF_HKEY_MAX_INDEX + 1];
164
165struct rte_i40evf_xstats_name_off {
166	char name[RTE_ETH_XSTATS_NAME_SIZE];
167	unsigned offset;
168};
169
170static const struct rte_i40evf_xstats_name_off rte_i40evf_stats_strings[] = {
171	{"rx_bytes", offsetof(struct i40e_eth_stats, rx_bytes)},
172	{"rx_unicast_packets", offsetof(struct i40e_eth_stats, rx_unicast)},
173	{"rx_multicast_packets", offsetof(struct i40e_eth_stats, rx_multicast)},
174	{"rx_broadcast_packets", offsetof(struct i40e_eth_stats, rx_broadcast)},
175	{"rx_dropped_packets", offsetof(struct i40e_eth_stats, rx_discards)},
176	{"rx_unknown_protocol_packets", offsetof(struct i40e_eth_stats,
177		rx_unknown_protocol)},
178	{"tx_bytes", offsetof(struct i40e_eth_stats, tx_bytes)},
179	{"tx_unicast_packets", offsetof(struct i40e_eth_stats, tx_bytes)},
180	{"tx_multicast_packets", offsetof(struct i40e_eth_stats, tx_bytes)},
181	{"tx_broadcast_packets", offsetof(struct i40e_eth_stats, tx_bytes)},
182	{"tx_dropped_packets", offsetof(struct i40e_eth_stats, tx_bytes)},
183	{"tx_error_packets", offsetof(struct i40e_eth_stats, tx_bytes)},
184};
185
186#define I40EVF_NB_XSTATS (sizeof(rte_i40evf_stats_strings) / \
187		sizeof(rte_i40evf_stats_strings[0]))
188
189static const struct eth_dev_ops i40evf_eth_dev_ops = {
190	.dev_configure        = i40evf_dev_configure,
191	.dev_start            = i40evf_dev_start,
192	.dev_stop             = i40evf_dev_stop,
193	.promiscuous_enable   = i40evf_dev_promiscuous_enable,
194	.promiscuous_disable  = i40evf_dev_promiscuous_disable,
195	.allmulticast_enable  = i40evf_dev_allmulticast_enable,
196	.allmulticast_disable = i40evf_dev_allmulticast_disable,
197	.link_update          = i40evf_dev_link_update,
198	.stats_get            = i40evf_dev_stats_get,
199	.xstats_get           = i40evf_dev_xstats_get,
200	.xstats_get_names     = i40evf_dev_xstats_get_names,
201	.xstats_reset         = i40evf_dev_xstats_reset,
202	.dev_close            = i40evf_dev_close,
203	.dev_infos_get        = i40evf_dev_info_get,
204	.dev_supported_ptypes_get = i40e_dev_supported_ptypes_get,
205	.vlan_filter_set      = i40evf_vlan_filter_set,
206	.vlan_offload_set     = i40evf_vlan_offload_set,
207	.vlan_pvid_set        = i40evf_vlan_pvid_set,
208	.rx_queue_start       = i40evf_dev_rx_queue_start,
209	.rx_queue_stop        = i40evf_dev_rx_queue_stop,
210	.tx_queue_start       = i40evf_dev_tx_queue_start,
211	.tx_queue_stop        = i40evf_dev_tx_queue_stop,
212	.rx_queue_setup       = i40e_dev_rx_queue_setup,
213	.rx_queue_release     = i40e_dev_rx_queue_release,
214	.rx_queue_intr_enable = i40evf_dev_rx_queue_intr_enable,
215	.rx_queue_intr_disable = i40evf_dev_rx_queue_intr_disable,
216	.rx_descriptor_done   = i40e_dev_rx_descriptor_done,
217	.tx_queue_setup       = i40e_dev_tx_queue_setup,
218	.tx_queue_release     = i40e_dev_tx_queue_release,
219	.rx_queue_count       = i40e_dev_rx_queue_count,
220	.rxq_info_get         = i40e_rxq_info_get,
221	.txq_info_get         = i40e_txq_info_get,
222	.mac_addr_add	      = i40evf_add_mac_addr,
223	.mac_addr_remove      = i40evf_del_mac_addr,
224	.reta_update          = i40evf_dev_rss_reta_update,
225	.reta_query           = i40evf_dev_rss_reta_query,
226	.rss_hash_update      = i40evf_dev_rss_hash_update,
227	.rss_hash_conf_get    = i40evf_dev_rss_hash_conf_get,
228};
229
230/*
231 * Read data in admin queue to get msg from pf driver
232 */
233static enum i40evf_aq_result
234i40evf_read_pfmsg(struct rte_eth_dev *dev, struct i40evf_arq_msg_info *data)
235{
236	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
237	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
238	struct i40e_arq_event_info event;
239	enum i40e_virtchnl_ops opcode;
240	enum i40e_status_code retval;
241	int ret;
242	enum i40evf_aq_result result = I40EVF_MSG_NON;
243
244	event.buf_len = data->buf_len;
245	event.msg_buf = data->msg;
246	ret = i40e_clean_arq_element(hw, &event, NULL);
247	/* Can't read any msg from adminQ */
248	if (ret) {
249		if (ret != I40E_ERR_ADMIN_QUEUE_NO_WORK)
250			result = I40EVF_MSG_ERR;
251		return result;
252	}
253
254	opcode = (enum i40e_virtchnl_ops)rte_le_to_cpu_32(event.desc.cookie_high);
255	retval = (enum i40e_status_code)rte_le_to_cpu_32(event.desc.cookie_low);
256	/* pf sys event */
257	if (opcode == I40E_VIRTCHNL_OP_EVENT) {
258		struct i40e_virtchnl_pf_event *vpe =
259			(struct i40e_virtchnl_pf_event *)event.msg_buf;
260
261		result = I40EVF_MSG_SYS;
262		switch (vpe->event) {
263		case I40E_VIRTCHNL_EVENT_LINK_CHANGE:
264			vf->link_up =
265				vpe->event_data.link_event.link_status;
266			vf->link_speed =
267				vpe->event_data.link_event.link_speed;
268			vf->pend_msg |= PFMSG_LINK_CHANGE;
269			PMD_DRV_LOG(INFO, "Link status update:%s",
270				    vf->link_up ? "up" : "down");
271			break;
272		case I40E_VIRTCHNL_EVENT_RESET_IMPENDING:
273			vf->vf_reset = true;
274			vf->pend_msg |= PFMSG_RESET_IMPENDING;
275			PMD_DRV_LOG(INFO, "vf is reseting");
276			break;
277		case I40E_VIRTCHNL_EVENT_PF_DRIVER_CLOSE:
278			vf->dev_closed = true;
279			vf->pend_msg |= PFMSG_DRIVER_CLOSE;
280			PMD_DRV_LOG(INFO, "PF driver closed");
281			break;
282		default:
283			PMD_DRV_LOG(ERR, "%s: Unknown event %d from pf",
284				    __func__, vpe->event);
285		}
286	} else {
287		/* async reply msg on command issued by vf previously */
288		result = I40EVF_MSG_CMD;
289		/* Actual data length read from PF */
290		data->msg_len = event.msg_len;
291	}
292
293	data->result = retval;
294	data->ops = opcode;
295
296	return result;
297}
298
299/**
300 * clear current command. Only call in case execute
301 * _atomic_set_cmd successfully.
302 */
303static inline void
304_clear_cmd(struct i40e_vf *vf)
305{
306	rte_wmb();
307	vf->pend_cmd = I40E_VIRTCHNL_OP_UNKNOWN;
308}
309
310/*
311 * Check there is pending cmd in execution. If none, set new command.
312 */
313static inline int
314_atomic_set_cmd(struct i40e_vf *vf, enum i40e_virtchnl_ops ops)
315{
316	int ret = rte_atomic32_cmpset(&vf->pend_cmd,
317			I40E_VIRTCHNL_OP_UNKNOWN, ops);
318
319	if (!ret)
320		PMD_DRV_LOG(ERR, "There is incomplete cmd %d", vf->pend_cmd);
321
322	return !ret;
323}
324
325#define MAX_TRY_TIMES 200
326#define ASQ_DELAY_MS  10
327
328static int
329i40evf_execute_vf_cmd(struct rte_eth_dev *dev, struct vf_cmd_info *args)
330{
331	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
332	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
333	struct i40evf_arq_msg_info info;
334	enum i40evf_aq_result ret;
335	int err, i = 0;
336
337	if (_atomic_set_cmd(vf, args->ops))
338		return -1;
339
340	info.msg = args->out_buffer;
341	info.buf_len = args->out_size;
342	info.ops = I40E_VIRTCHNL_OP_UNKNOWN;
343	info.result = I40E_SUCCESS;
344
345	err = i40e_aq_send_msg_to_pf(hw, args->ops, I40E_SUCCESS,
346		     args->in_args, args->in_args_size, NULL);
347	if (err) {
348		PMD_DRV_LOG(ERR, "fail to send cmd %d", args->ops);
349		_clear_cmd(vf);
350		return err;
351	}
352
353	switch (args->ops) {
354	case I40E_VIRTCHNL_OP_RESET_VF:
355		/*no need to process in this function */
356		err = 0;
357		break;
358	case I40E_VIRTCHNL_OP_VERSION:
359	case I40E_VIRTCHNL_OP_GET_VF_RESOURCES:
360		/* for init adminq commands, need to poll the response */
361		err = -1;
362		do {
363			ret = i40evf_read_pfmsg(dev, &info);
364			if (ret == I40EVF_MSG_CMD) {
365				err = 0;
366				break;
367			} else if (ret == I40EVF_MSG_ERR)
368				break;
369			rte_delay_ms(ASQ_DELAY_MS);
370			/* If don't read msg or read sys event, continue */
371		} while (i++ < MAX_TRY_TIMES);
372		_clear_cmd(vf);
373		break;
374
375	default:
376		/* for other adminq in running time, waiting the cmd done flag */
377		err = -1;
378		do {
379			if (vf->pend_cmd == I40E_VIRTCHNL_OP_UNKNOWN) {
380				err = 0;
381				break;
382			}
383			rte_delay_ms(ASQ_DELAY_MS);
384			/* If don't read msg or read sys event, continue */
385		} while (i++ < MAX_TRY_TIMES);
386		break;
387	}
388
389	return err | vf->cmd_retval;
390}
391
392/*
393 * Check API version with sync wait until version read or fail from admin queue
394 */
395static int
396i40evf_check_api_version(struct rte_eth_dev *dev)
397{
398	struct i40e_virtchnl_version_info version, *pver;
399	int err;
400	struct vf_cmd_info args;
401	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
402
403	version.major = I40E_VIRTCHNL_VERSION_MAJOR;
404	version.minor = I40E_VIRTCHNL_VERSION_MINOR;
405
406	args.ops = I40E_VIRTCHNL_OP_VERSION;
407	args.in_args = (uint8_t *)&version;
408	args.in_args_size = sizeof(version);
409	args.out_buffer = vf->aq_resp;
410	args.out_size = I40E_AQ_BUF_SZ;
411
412	err = i40evf_execute_vf_cmd(dev, &args);
413	if (err) {
414		PMD_INIT_LOG(ERR, "fail to execute command OP_VERSION");
415		return err;
416	}
417
418	pver = (struct i40e_virtchnl_version_info *)args.out_buffer;
419	vf->version_major = pver->major;
420	vf->version_minor = pver->minor;
421	if (vf->version_major == I40E_DPDK_VERSION_MAJOR)
422		PMD_DRV_LOG(INFO, "Peer is DPDK PF host");
423	else if ((vf->version_major == I40E_VIRTCHNL_VERSION_MAJOR) &&
424		(vf->version_minor <= I40E_VIRTCHNL_VERSION_MINOR))
425		PMD_DRV_LOG(INFO, "Peer is Linux PF host");
426	else {
427		PMD_INIT_LOG(ERR, "PF/VF API version mismatch:(%u.%u)-(%u.%u)",
428					vf->version_major, vf->version_minor,
429						I40E_VIRTCHNL_VERSION_MAJOR,
430						I40E_VIRTCHNL_VERSION_MINOR);
431		return -1;
432	}
433
434	return 0;
435}
436
437static int
438i40evf_get_vf_resource(struct rte_eth_dev *dev)
439{
440	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
441	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
442	int err;
443	struct vf_cmd_info args;
444	uint32_t caps, len;
445
446	args.ops = I40E_VIRTCHNL_OP_GET_VF_RESOURCES;
447	args.out_buffer = vf->aq_resp;
448	args.out_size = I40E_AQ_BUF_SZ;
449	if (PF_IS_V11(vf)) {
450		caps = I40E_VIRTCHNL_VF_OFFLOAD_L2 |
451		       I40E_VIRTCHNL_VF_OFFLOAD_RSS_AQ |
452		       I40E_VIRTCHNL_VF_OFFLOAD_RSS_REG |
453		       I40E_VIRTCHNL_VF_OFFLOAD_VLAN |
454		       I40E_VIRTCHNL_VF_OFFLOAD_RX_POLLING;
455		args.in_args = (uint8_t *)&caps;
456		args.in_args_size = sizeof(caps);
457	} else {
458		args.in_args = NULL;
459		args.in_args_size = 0;
460	}
461	err = i40evf_execute_vf_cmd(dev, &args);
462
463	if (err) {
464		PMD_DRV_LOG(ERR, "fail to execute command OP_GET_VF_RESOURCE");
465		return err;
466	}
467
468	len =  sizeof(struct i40e_virtchnl_vf_resource) +
469		I40E_MAX_VF_VSI * sizeof(struct i40e_virtchnl_vsi_resource);
470
471	(void)rte_memcpy(vf->vf_res, args.out_buffer,
472			RTE_MIN(args.out_size, len));
473	i40e_vf_parse_hw_config(hw, vf->vf_res);
474
475	return 0;
476}
477
478static int
479i40evf_config_promisc(struct rte_eth_dev *dev,
480		      bool enable_unicast,
481		      bool enable_multicast)
482{
483	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
484	int err;
485	struct vf_cmd_info args;
486	struct i40e_virtchnl_promisc_info promisc;
487
488	promisc.flags = 0;
489	promisc.vsi_id = vf->vsi_res->vsi_id;
490
491	if (enable_unicast)
492		promisc.flags |= I40E_FLAG_VF_UNICAST_PROMISC;
493
494	if (enable_multicast)
495		promisc.flags |= I40E_FLAG_VF_MULTICAST_PROMISC;
496
497	args.ops = I40E_VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE;
498	args.in_args = (uint8_t *)&promisc;
499	args.in_args_size = sizeof(promisc);
500	args.out_buffer = vf->aq_resp;
501	args.out_size = I40E_AQ_BUF_SZ;
502
503	err = i40evf_execute_vf_cmd(dev, &args);
504
505	if (err)
506		PMD_DRV_LOG(ERR, "fail to execute command "
507			    "CONFIG_PROMISCUOUS_MODE");
508	return err;
509}
510
511/* Configure vlan and double vlan offload. Use flag to specify which part to configure */
512static int
513i40evf_config_vlan_offload(struct rte_eth_dev *dev,
514				bool enable_vlan_strip)
515{
516	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
517	int err;
518	struct vf_cmd_info args;
519	struct i40e_virtchnl_vlan_offload_info offload;
520
521	offload.vsi_id = vf->vsi_res->vsi_id;
522	offload.enable_vlan_strip = enable_vlan_strip;
523
524	args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CFG_VLAN_OFFLOAD;
525	args.in_args = (uint8_t *)&offload;
526	args.in_args_size = sizeof(offload);
527	args.out_buffer = vf->aq_resp;
528	args.out_size = I40E_AQ_BUF_SZ;
529
530	err = i40evf_execute_vf_cmd(dev, &args);
531	if (err)
532		PMD_DRV_LOG(ERR, "fail to execute command CFG_VLAN_OFFLOAD");
533
534	return err;
535}
536
537static int
538i40evf_config_vlan_pvid(struct rte_eth_dev *dev,
539				struct i40e_vsi_vlan_pvid_info *info)
540{
541	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
542	int err;
543	struct vf_cmd_info args;
544	struct i40e_virtchnl_pvid_info tpid_info;
545
546	if (info == NULL) {
547		PMD_DRV_LOG(ERR, "invalid parameters");
548		return I40E_ERR_PARAM;
549	}
550
551	memset(&tpid_info, 0, sizeof(tpid_info));
552	tpid_info.vsi_id = vf->vsi_res->vsi_id;
553	(void)rte_memcpy(&tpid_info.info, info, sizeof(*info));
554
555	args.ops = (enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CFG_VLAN_PVID;
556	args.in_args = (uint8_t *)&tpid_info;
557	args.in_args_size = sizeof(tpid_info);
558	args.out_buffer = vf->aq_resp;
559	args.out_size = I40E_AQ_BUF_SZ;
560
561	err = i40evf_execute_vf_cmd(dev, &args);
562	if (err)
563		PMD_DRV_LOG(ERR, "fail to execute command CFG_VLAN_PVID");
564
565	return err;
566}
567
568static void
569i40evf_fill_virtchnl_vsi_txq_info(struct i40e_virtchnl_txq_info *txq_info,
570				  uint16_t vsi_id,
571				  uint16_t queue_id,
572				  uint16_t nb_txq,
573				  struct i40e_tx_queue *txq)
574{
575	txq_info->vsi_id = vsi_id;
576	txq_info->queue_id = queue_id;
577	if (queue_id < nb_txq) {
578		txq_info->ring_len = txq->nb_tx_desc;
579		txq_info->dma_ring_addr = txq->tx_ring_phys_addr;
580	}
581}
582
583static void
584i40evf_fill_virtchnl_vsi_rxq_info(struct i40e_virtchnl_rxq_info *rxq_info,
585				  uint16_t vsi_id,
586				  uint16_t queue_id,
587				  uint16_t nb_rxq,
588				  uint32_t max_pkt_size,
589				  struct i40e_rx_queue *rxq)
590{
591	rxq_info->vsi_id = vsi_id;
592	rxq_info->queue_id = queue_id;
593	rxq_info->max_pkt_size = max_pkt_size;
594	if (queue_id < nb_rxq) {
595		rxq_info->ring_len = rxq->nb_rx_desc;
596		rxq_info->dma_ring_addr = rxq->rx_ring_phys_addr;
597		rxq_info->databuffer_size =
598			(rte_pktmbuf_data_room_size(rxq->mp) -
599				RTE_PKTMBUF_HEADROOM);
600	}
601}
602
603/* It configures VSI queues to co-work with Linux PF host */
604static int
605i40evf_configure_vsi_queues(struct rte_eth_dev *dev)
606{
607	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
608	struct i40e_rx_queue **rxq =
609		(struct i40e_rx_queue **)dev->data->rx_queues;
610	struct i40e_tx_queue **txq =
611		(struct i40e_tx_queue **)dev->data->tx_queues;
612	struct i40e_virtchnl_vsi_queue_config_info *vc_vqci;
613	struct i40e_virtchnl_queue_pair_info *vc_qpi;
614	struct vf_cmd_info args;
615	uint16_t i, nb_qp = vf->num_queue_pairs;
616	const uint32_t size =
617		I40E_VIRTCHNL_CONFIG_VSI_QUEUES_SIZE(vc_vqci, nb_qp);
618	uint8_t buff[size];
619	int ret;
620
621	memset(buff, 0, sizeof(buff));
622	vc_vqci = (struct i40e_virtchnl_vsi_queue_config_info *)buff;
623	vc_vqci->vsi_id = vf->vsi_res->vsi_id;
624	vc_vqci->num_queue_pairs = nb_qp;
625
626	for (i = 0, vc_qpi = vc_vqci->qpair; i < nb_qp; i++, vc_qpi++) {
627		i40evf_fill_virtchnl_vsi_txq_info(&vc_qpi->txq,
628			vc_vqci->vsi_id, i, dev->data->nb_tx_queues, txq[i]);
629		i40evf_fill_virtchnl_vsi_rxq_info(&vc_qpi->rxq,
630			vc_vqci->vsi_id, i, dev->data->nb_rx_queues,
631					vf->max_pkt_len, rxq[i]);
632	}
633	memset(&args, 0, sizeof(args));
634	args.ops = I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES;
635	args.in_args = (uint8_t *)vc_vqci;
636	args.in_args_size = size;
637	args.out_buffer = vf->aq_resp;
638	args.out_size = I40E_AQ_BUF_SZ;
639	ret = i40evf_execute_vf_cmd(dev, &args);
640	if (ret)
641		PMD_DRV_LOG(ERR, "Failed to execute command of "
642			"I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES\n");
643
644	return ret;
645}
646
647/* It configures VSI queues to co-work with DPDK PF host */
648static int
649i40evf_configure_vsi_queues_ext(struct rte_eth_dev *dev)
650{
651	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
652	struct i40e_rx_queue **rxq =
653		(struct i40e_rx_queue **)dev->data->rx_queues;
654	struct i40e_tx_queue **txq =
655		(struct i40e_tx_queue **)dev->data->tx_queues;
656	struct i40e_virtchnl_vsi_queue_config_ext_info *vc_vqcei;
657	struct i40e_virtchnl_queue_pair_ext_info *vc_qpei;
658	struct vf_cmd_info args;
659	uint16_t i, nb_qp = vf->num_queue_pairs;
660	const uint32_t size =
661		I40E_VIRTCHNL_CONFIG_VSI_QUEUES_SIZE(vc_vqcei, nb_qp);
662	uint8_t buff[size];
663	int ret;
664
665	memset(buff, 0, sizeof(buff));
666	vc_vqcei = (struct i40e_virtchnl_vsi_queue_config_ext_info *)buff;
667	vc_vqcei->vsi_id = vf->vsi_res->vsi_id;
668	vc_vqcei->num_queue_pairs = nb_qp;
669	vc_qpei = vc_vqcei->qpair;
670	for (i = 0; i < nb_qp; i++, vc_qpei++) {
671		i40evf_fill_virtchnl_vsi_txq_info(&vc_qpei->txq,
672			vc_vqcei->vsi_id, i, dev->data->nb_tx_queues, txq[i]);
673		i40evf_fill_virtchnl_vsi_rxq_info(&vc_qpei->rxq,
674			vc_vqcei->vsi_id, i, dev->data->nb_rx_queues,
675					vf->max_pkt_len, rxq[i]);
676		if (i < dev->data->nb_rx_queues)
677			/*
678			 * It adds extra info for configuring VSI queues, which
679			 * is needed to enable the configurable crc stripping
680			 * in VF.
681			 */
682			vc_qpei->rxq_ext.crcstrip =
683				dev->data->dev_conf.rxmode.hw_strip_crc;
684	}
685	memset(&args, 0, sizeof(args));
686	args.ops =
687		(enum i40e_virtchnl_ops)I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT;
688	args.in_args = (uint8_t *)vc_vqcei;
689	args.in_args_size = size;
690	args.out_buffer = vf->aq_resp;
691	args.out_size = I40E_AQ_BUF_SZ;
692	ret = i40evf_execute_vf_cmd(dev, &args);
693	if (ret)
694		PMD_DRV_LOG(ERR, "Failed to execute command of "
695			"I40E_VIRTCHNL_OP_CONFIG_VSI_QUEUES_EXT\n");
696
697	return ret;
698}
699
700static int
701i40evf_configure_queues(struct rte_eth_dev *dev)
702{
703	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
704
705	if (vf->version_major == I40E_DPDK_VERSION_MAJOR)
706		/* To support DPDK PF host */
707		return i40evf_configure_vsi_queues_ext(dev);
708	else
709		/* To support Linux PF host */
710		return i40evf_configure_vsi_queues(dev);
711}
712
713static int
714i40evf_config_irq_map(struct rte_eth_dev *dev)
715{
716	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
717	struct vf_cmd_info args;
718	uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_irq_map_info) + \
719		sizeof(struct i40e_virtchnl_vector_map)];
720	struct i40e_virtchnl_irq_map_info *map_info;
721	struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
722	uint32_t vector_id;
723	int i, err;
724
725	if (rte_intr_allow_others(intr_handle)) {
726		if (vf->version_major == I40E_DPDK_VERSION_MAJOR)
727			vector_id = I40EVF_VSI_DEFAULT_MSIX_INTR;
728		else
729			vector_id = I40EVF_VSI_DEFAULT_MSIX_INTR_LNX;
730	} else {
731		vector_id = I40E_MISC_VEC_ID;
732	}
733
734	map_info = (struct i40e_virtchnl_irq_map_info *)cmd_buffer;
735	map_info->num_vectors = 1;
736	map_info->vecmap[0].rxitr_idx = I40E_ITR_INDEX_DEFAULT;
737	map_info->vecmap[0].vsi_id = vf->vsi_res->vsi_id;
738	/* Alway use default dynamic MSIX interrupt */
739	map_info->vecmap[0].vector_id = vector_id;
740	/* Don't map any tx queue */
741	map_info->vecmap[0].txq_map = 0;
742	map_info->vecmap[0].rxq_map = 0;
743	for (i = 0; i < dev->data->nb_rx_queues; i++) {
744		map_info->vecmap[0].rxq_map |= 1 << i;
745		if (rte_intr_dp_is_en(intr_handle))
746			intr_handle->intr_vec[i] = vector_id;
747	}
748
749	args.ops = I40E_VIRTCHNL_OP_CONFIG_IRQ_MAP;
750	args.in_args = (u8 *)cmd_buffer;
751	args.in_args_size = sizeof(cmd_buffer);
752	args.out_buffer = vf->aq_resp;
753	args.out_size = I40E_AQ_BUF_SZ;
754	err = i40evf_execute_vf_cmd(dev, &args);
755	if (err)
756		PMD_DRV_LOG(ERR, "fail to execute command OP_ENABLE_QUEUES");
757
758	return err;
759}
760
761static int
762i40evf_switch_queue(struct rte_eth_dev *dev, bool isrx, uint16_t qid,
763				bool on)
764{
765	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
766	struct i40e_virtchnl_queue_select queue_select;
767	int err;
768	struct vf_cmd_info args;
769	memset(&queue_select, 0, sizeof(queue_select));
770	queue_select.vsi_id = vf->vsi_res->vsi_id;
771
772	if (isrx)
773		queue_select.rx_queues |= 1 << qid;
774	else
775		queue_select.tx_queues |= 1 << qid;
776
777	if (on)
778		args.ops = I40E_VIRTCHNL_OP_ENABLE_QUEUES;
779	else
780		args.ops = I40E_VIRTCHNL_OP_DISABLE_QUEUES;
781	args.in_args = (u8 *)&queue_select;
782	args.in_args_size = sizeof(queue_select);
783	args.out_buffer = vf->aq_resp;
784	args.out_size = I40E_AQ_BUF_SZ;
785	err = i40evf_execute_vf_cmd(dev, &args);
786	if (err)
787		PMD_DRV_LOG(ERR, "fail to switch %s %u %s",
788			    isrx ? "RX" : "TX", qid, on ? "on" : "off");
789
790	return err;
791}
792
793static int
794i40evf_start_queues(struct rte_eth_dev *dev)
795{
796	struct rte_eth_dev_data *dev_data = dev->data;
797	int i;
798	struct i40e_rx_queue *rxq;
799	struct i40e_tx_queue *txq;
800
801	for (i = 0; i < dev->data->nb_rx_queues; i++) {
802		rxq = dev_data->rx_queues[i];
803		if (rxq->rx_deferred_start)
804			continue;
805		if (i40evf_dev_rx_queue_start(dev, i) != 0) {
806			PMD_DRV_LOG(ERR, "Fail to start queue %u", i);
807			return -1;
808		}
809	}
810
811	for (i = 0; i < dev->data->nb_tx_queues; i++) {
812		txq = dev_data->tx_queues[i];
813		if (txq->tx_deferred_start)
814			continue;
815		if (i40evf_dev_tx_queue_start(dev, i) != 0) {
816			PMD_DRV_LOG(ERR, "Fail to start queue %u", i);
817			return -1;
818		}
819	}
820
821	return 0;
822}
823
824static int
825i40evf_stop_queues(struct rte_eth_dev *dev)
826{
827	int i;
828
829	/* Stop TX queues first */
830	for (i = 0; i < dev->data->nb_tx_queues; i++) {
831		if (i40evf_dev_tx_queue_stop(dev, i) != 0) {
832			PMD_DRV_LOG(ERR, "Fail to stop queue %u", i);
833			return -1;
834		}
835	}
836
837	/* Then stop RX queues */
838	for (i = 0; i < dev->data->nb_rx_queues; i++) {
839		if (i40evf_dev_rx_queue_stop(dev, i) != 0) {
840			PMD_DRV_LOG(ERR, "Fail to stop queue %u", i);
841			return -1;
842		}
843	}
844
845	return 0;
846}
847
848static void
849i40evf_add_mac_addr(struct rte_eth_dev *dev,
850		    struct ether_addr *addr,
851		    __rte_unused uint32_t index,
852		    __rte_unused uint32_t pool)
853{
854	struct i40e_virtchnl_ether_addr_list *list;
855	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
856	uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_ether_addr_list) + \
857			sizeof(struct i40e_virtchnl_ether_addr)];
858	int err;
859	struct vf_cmd_info args;
860
861	if (i40e_validate_mac_addr(addr->addr_bytes) != I40E_SUCCESS) {
862		PMD_DRV_LOG(ERR, "Invalid mac:%x:%x:%x:%x:%x:%x",
863			    addr->addr_bytes[0], addr->addr_bytes[1],
864			    addr->addr_bytes[2], addr->addr_bytes[3],
865			    addr->addr_bytes[4], addr->addr_bytes[5]);
866		return;
867	}
868
869	list = (struct i40e_virtchnl_ether_addr_list *)cmd_buffer;
870	list->vsi_id = vf->vsi_res->vsi_id;
871	list->num_elements = 1;
872	(void)rte_memcpy(list->list[0].addr, addr->addr_bytes,
873					sizeof(addr->addr_bytes));
874
875	args.ops = I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS;
876	args.in_args = cmd_buffer;
877	args.in_args_size = sizeof(cmd_buffer);
878	args.out_buffer = vf->aq_resp;
879	args.out_size = I40E_AQ_BUF_SZ;
880	err = i40evf_execute_vf_cmd(dev, &args);
881	if (err)
882		PMD_DRV_LOG(ERR, "fail to execute command "
883			    "OP_ADD_ETHER_ADDRESS");
884
885	return;
886}
887
888static void
889i40evf_del_mac_addr(struct rte_eth_dev *dev, uint32_t index)
890{
891	struct i40e_virtchnl_ether_addr_list *list;
892	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
893	struct rte_eth_dev_data *data = dev->data;
894	struct ether_addr *addr;
895	uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_ether_addr_list) + \
896			sizeof(struct i40e_virtchnl_ether_addr)];
897	int err;
898	struct vf_cmd_info args;
899
900	addr = &(data->mac_addrs[index]);
901
902	if (i40e_validate_mac_addr(addr->addr_bytes) != I40E_SUCCESS) {
903		PMD_DRV_LOG(ERR, "Invalid mac:%x-%x-%x-%x-%x-%x",
904			    addr->addr_bytes[0], addr->addr_bytes[1],
905			    addr->addr_bytes[2], addr->addr_bytes[3],
906			    addr->addr_bytes[4], addr->addr_bytes[5]);
907		return;
908	}
909
910	list = (struct i40e_virtchnl_ether_addr_list *)cmd_buffer;
911	list->vsi_id = vf->vsi_res->vsi_id;
912	list->num_elements = 1;
913	(void)rte_memcpy(list->list[0].addr, addr->addr_bytes,
914			sizeof(addr->addr_bytes));
915
916	args.ops = I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS;
917	args.in_args = cmd_buffer;
918	args.in_args_size = sizeof(cmd_buffer);
919	args.out_buffer = vf->aq_resp;
920	args.out_size = I40E_AQ_BUF_SZ;
921	err = i40evf_execute_vf_cmd(dev, &args);
922	if (err)
923		PMD_DRV_LOG(ERR, "fail to execute command "
924			    "OP_DEL_ETHER_ADDRESS");
925	return;
926}
927
928static int
929i40evf_update_stats(struct rte_eth_dev *dev, struct i40e_eth_stats **pstats)
930{
931	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
932	struct i40e_virtchnl_queue_select q_stats;
933	int err;
934	struct vf_cmd_info args;
935
936	memset(&q_stats, 0, sizeof(q_stats));
937	q_stats.vsi_id = vf->vsi_res->vsi_id;
938	args.ops = I40E_VIRTCHNL_OP_GET_STATS;
939	args.in_args = (u8 *)&q_stats;
940	args.in_args_size = sizeof(q_stats);
941	args.out_buffer = vf->aq_resp;
942	args.out_size = I40E_AQ_BUF_SZ;
943
944	err = i40evf_execute_vf_cmd(dev, &args);
945	if (err) {
946		PMD_DRV_LOG(ERR, "fail to execute command OP_GET_STATS");
947		*pstats = NULL;
948		return err;
949	}
950	*pstats = (struct i40e_eth_stats *)args.out_buffer;
951	return 0;
952}
953
954static int
955i40evf_get_statics(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
956{
957	int ret;
958	struct i40e_eth_stats *pstats = NULL;
959
960	ret = i40evf_update_stats(dev, &pstats);
961	if (ret != 0)
962		return 0;
963
964	stats->ipackets = pstats->rx_unicast + pstats->rx_multicast +
965						pstats->rx_broadcast;
966	stats->opackets = pstats->tx_broadcast + pstats->tx_multicast +
967						pstats->tx_unicast;
968	stats->ierrors = pstats->rx_discards;
969	stats->oerrors = pstats->tx_errors + pstats->tx_discards;
970	stats->ibytes = pstats->rx_bytes;
971	stats->obytes = pstats->tx_bytes;
972
973	return 0;
974}
975
976static void
977i40evf_dev_xstats_reset(struct rte_eth_dev *dev)
978{
979	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
980	struct i40e_eth_stats *pstats = NULL;
981
982	/* read stat values to clear hardware registers */
983	i40evf_update_stats(dev, &pstats);
984
985	/* set stats offset base on current values */
986	vf->vsi.eth_stats_offset = vf->vsi.eth_stats;
987}
988
989static int i40evf_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
990				      struct rte_eth_xstat_name *xstats_names,
991				      __rte_unused unsigned limit)
992{
993	unsigned i;
994
995	if (xstats_names != NULL)
996		for (i = 0; i < I40EVF_NB_XSTATS; i++) {
997			snprintf(xstats_names[i].name,
998				sizeof(xstats_names[i].name),
999				"%s", rte_i40evf_stats_strings[i].name);
1000		}
1001	return I40EVF_NB_XSTATS;
1002}
1003
1004static int i40evf_dev_xstats_get(struct rte_eth_dev *dev,
1005				 struct rte_eth_xstat *xstats, unsigned n)
1006{
1007	int ret;
1008	unsigned i;
1009	struct i40e_eth_stats *pstats = NULL;
1010
1011	if (n < I40EVF_NB_XSTATS)
1012		return I40EVF_NB_XSTATS;
1013
1014	ret = i40evf_update_stats(dev, &pstats);
1015	if (ret != 0)
1016		return 0;
1017
1018	if (!xstats)
1019		return 0;
1020
1021	/* loop over xstats array and values from pstats */
1022	for (i = 0; i < I40EVF_NB_XSTATS; i++) {
1023		xstats[i].id = i;
1024		xstats[i].value = *(uint64_t *)(((char *)pstats) +
1025			rte_i40evf_stats_strings[i].offset);
1026	}
1027
1028	return I40EVF_NB_XSTATS;
1029}
1030
1031static int
1032i40evf_add_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
1033{
1034	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1035	struct i40e_virtchnl_vlan_filter_list *vlan_list;
1036	uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_vlan_filter_list) +
1037							sizeof(uint16_t)];
1038	int err;
1039	struct vf_cmd_info args;
1040
1041	vlan_list = (struct i40e_virtchnl_vlan_filter_list *)cmd_buffer;
1042	vlan_list->vsi_id = vf->vsi_res->vsi_id;
1043	vlan_list->num_elements = 1;
1044	vlan_list->vlan_id[0] = vlanid;
1045
1046	args.ops = I40E_VIRTCHNL_OP_ADD_VLAN;
1047	args.in_args = (u8 *)&cmd_buffer;
1048	args.in_args_size = sizeof(cmd_buffer);
1049	args.out_buffer = vf->aq_resp;
1050	args.out_size = I40E_AQ_BUF_SZ;
1051	err = i40evf_execute_vf_cmd(dev, &args);
1052	if (err)
1053		PMD_DRV_LOG(ERR, "fail to execute command OP_ADD_VLAN");
1054
1055	return err;
1056}
1057
1058static int
1059i40evf_del_vlan(struct rte_eth_dev *dev, uint16_t vlanid)
1060{
1061	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1062	struct i40e_virtchnl_vlan_filter_list *vlan_list;
1063	uint8_t cmd_buffer[sizeof(struct i40e_virtchnl_vlan_filter_list) +
1064							sizeof(uint16_t)];
1065	int err;
1066	struct vf_cmd_info args;
1067
1068	vlan_list = (struct i40e_virtchnl_vlan_filter_list *)cmd_buffer;
1069	vlan_list->vsi_id = vf->vsi_res->vsi_id;
1070	vlan_list->num_elements = 1;
1071	vlan_list->vlan_id[0] = vlanid;
1072
1073	args.ops = I40E_VIRTCHNL_OP_DEL_VLAN;
1074	args.in_args = (u8 *)&cmd_buffer;
1075	args.in_args_size = sizeof(cmd_buffer);
1076	args.out_buffer = vf->aq_resp;
1077	args.out_size = I40E_AQ_BUF_SZ;
1078	err = i40evf_execute_vf_cmd(dev, &args);
1079	if (err)
1080		PMD_DRV_LOG(ERR, "fail to execute command OP_DEL_VLAN");
1081
1082	return err;
1083}
1084
1085static const struct rte_pci_id pci_id_i40evf_map[] = {
1086	{ RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_VF) },
1087	{ RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_VF_HV) },
1088	{ RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_A0_VF) },
1089	{ RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_VF) },
1090	{ RTE_PCI_DEVICE(I40E_INTEL_VENDOR_ID, I40E_DEV_ID_X722_VF_HV) },
1091	{ .vendor_id = 0, /* sentinel */ },
1092};
1093
1094static inline int
1095i40evf_dev_atomic_write_link_status(struct rte_eth_dev *dev,
1096				    struct rte_eth_link *link)
1097{
1098	struct rte_eth_link *dst = &(dev->data->dev_link);
1099	struct rte_eth_link *src = link;
1100
1101	if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
1102					*(uint64_t *)src) == 0)
1103		return -1;
1104
1105	return 0;
1106}
1107
1108/* Disable IRQ0 */
1109static inline void
1110i40evf_disable_irq0(struct i40e_hw *hw)
1111{
1112	/* Disable all interrupt types */
1113	I40E_WRITE_REG(hw, I40E_VFINT_ICR0_ENA1, 0);
1114	I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
1115		       I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
1116	I40EVF_WRITE_FLUSH(hw);
1117}
1118
1119/* Enable IRQ0 */
1120static inline void
1121i40evf_enable_irq0(struct i40e_hw *hw)
1122{
1123	/* Enable admin queue interrupt trigger */
1124	uint32_t val;
1125
1126	i40evf_disable_irq0(hw);
1127	val = I40E_READ_REG(hw, I40E_VFINT_ICR0_ENA1);
1128	val |= I40E_VFINT_ICR0_ENA1_ADMINQ_MASK |
1129		I40E_VFINT_ICR0_ENA1_LINK_STAT_CHANGE_MASK;
1130	I40E_WRITE_REG(hw, I40E_VFINT_ICR0_ENA1, val);
1131
1132	I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
1133		I40E_VFINT_DYN_CTL01_INTENA_MASK |
1134		I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
1135		I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
1136
1137	I40EVF_WRITE_FLUSH(hw);
1138}
1139
1140static int
1141i40evf_reset_vf(struct i40e_hw *hw)
1142{
1143	int i, reset;
1144
1145	if (i40e_vf_reset(hw) != I40E_SUCCESS) {
1146		PMD_INIT_LOG(ERR, "Reset VF NIC failed");
1147		return -1;
1148	}
1149	/**
1150	  * After issuing vf reset command to pf, pf won't necessarily
1151	  * reset vf, it depends on what state it exactly is. If it's not
1152	  * initialized yet, it won't have vf reset since it's in a certain
1153	  * state. If not, it will try to reset. Even vf is reset, pf will
1154	  * set I40E_VFGEN_RSTAT to COMPLETE first, then wait 10ms and set
1155	  * it to ACTIVE. In this duration, vf may not catch the moment that
1156	  * COMPLETE is set. So, for vf, we'll try to wait a long time.
1157	  */
1158	rte_delay_ms(200);
1159
1160	for (i = 0; i < MAX_RESET_WAIT_CNT; i++) {
1161		reset = rd32(hw, I40E_VFGEN_RSTAT) &
1162			I40E_VFGEN_RSTAT_VFR_STATE_MASK;
1163		reset = reset >> I40E_VFGEN_RSTAT_VFR_STATE_SHIFT;
1164		if (I40E_VFR_COMPLETED == reset || I40E_VFR_VFACTIVE == reset)
1165			break;
1166		else
1167			rte_delay_ms(50);
1168	}
1169
1170	if (i >= MAX_RESET_WAIT_CNT) {
1171		PMD_INIT_LOG(ERR, "Reset VF NIC failed");
1172		return -1;
1173	}
1174
1175	return 0;
1176}
1177
1178static int
1179i40evf_init_vf(struct rte_eth_dev *dev)
1180{
1181	int i, err, bufsz;
1182	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1183	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1184	struct ether_addr *p_mac_addr;
1185	uint16_t interval =
1186		i40e_calc_itr_interval(I40E_QUEUE_ITR_INTERVAL_MAX);
1187
1188	vf->adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1189	vf->dev_data = dev->data;
1190	err = i40e_set_mac_type(hw);
1191	if (err) {
1192		PMD_INIT_LOG(ERR, "set_mac_type failed: %d", err);
1193		goto err;
1194	}
1195
1196	i40e_init_adminq_parameter(hw);
1197	err = i40e_init_adminq(hw);
1198	if (err) {
1199		PMD_INIT_LOG(ERR, "init_adminq failed: %d", err);
1200		goto err;
1201	}
1202
1203	/* Reset VF and wait until it's complete */
1204	if (i40evf_reset_vf(hw)) {
1205		PMD_INIT_LOG(ERR, "reset NIC failed");
1206		goto err_aq;
1207	}
1208
1209	/* VF reset, shutdown admin queue and initialize again */
1210	if (i40e_shutdown_adminq(hw) != I40E_SUCCESS) {
1211		PMD_INIT_LOG(ERR, "i40e_shutdown_adminq failed");
1212		return -1;
1213	}
1214
1215	i40e_init_adminq_parameter(hw);
1216	if (i40e_init_adminq(hw) != I40E_SUCCESS) {
1217		PMD_INIT_LOG(ERR, "init_adminq failed");
1218		return -1;
1219	}
1220	vf->aq_resp = rte_zmalloc("vf_aq_resp", I40E_AQ_BUF_SZ, 0);
1221	if (!vf->aq_resp) {
1222		PMD_INIT_LOG(ERR, "unable to allocate vf_aq_resp memory");
1223			goto err_aq;
1224	}
1225	if (i40evf_check_api_version(dev) != 0) {
1226		PMD_INIT_LOG(ERR, "check_api version failed");
1227		goto err_aq;
1228	}
1229	bufsz = sizeof(struct i40e_virtchnl_vf_resource) +
1230		(I40E_MAX_VF_VSI * sizeof(struct i40e_virtchnl_vsi_resource));
1231	vf->vf_res = rte_zmalloc("vf_res", bufsz, 0);
1232	if (!vf->vf_res) {
1233		PMD_INIT_LOG(ERR, "unable to allocate vf_res memory");
1234			goto err_aq;
1235	}
1236
1237	if (i40evf_get_vf_resource(dev) != 0) {
1238		PMD_INIT_LOG(ERR, "i40evf_get_vf_config failed");
1239		goto err_alloc;
1240	}
1241
1242	/* got VF config message back from PF, now we can parse it */
1243	for (i = 0; i < vf->vf_res->num_vsis; i++) {
1244		if (vf->vf_res->vsi_res[i].vsi_type == I40E_VSI_SRIOV)
1245			vf->vsi_res = &vf->vf_res->vsi_res[i];
1246	}
1247
1248	if (!vf->vsi_res) {
1249		PMD_INIT_LOG(ERR, "no LAN VSI found");
1250		goto err_alloc;
1251	}
1252
1253	if (hw->mac.type == I40E_MAC_X722_VF)
1254		vf->flags = I40E_FLAG_RSS_AQ_CAPABLE;
1255	vf->vsi.vsi_id = vf->vsi_res->vsi_id;
1256	vf->vsi.type = vf->vsi_res->vsi_type;
1257	vf->vsi.nb_qps = vf->vsi_res->num_queue_pairs;
1258	vf->vsi.adapter = I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1259
1260	/* Store the MAC address configured by host, or generate random one */
1261	p_mac_addr = (struct ether_addr *)(vf->vsi_res->default_mac_addr);
1262	if (is_valid_assigned_ether_addr(p_mac_addr)) /* Configured by host */
1263		ether_addr_copy(p_mac_addr, (struct ether_addr *)hw->mac.addr);
1264	else
1265		eth_random_addr(hw->mac.addr); /* Generate a random one */
1266
1267	/* If the PF host is not DPDK, set the interval of ITR0 to max*/
1268	if (vf->version_major != I40E_DPDK_VERSION_MAJOR) {
1269		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
1270			       (I40E_ITR_INDEX_DEFAULT <<
1271				I40E_VFINT_DYN_CTL0_ITR_INDX_SHIFT) |
1272			       (interval <<
1273				I40E_VFINT_DYN_CTL0_INTERVAL_SHIFT));
1274		I40EVF_WRITE_FLUSH(hw);
1275	}
1276
1277	return 0;
1278
1279err_alloc:
1280	rte_free(vf->vf_res);
1281err_aq:
1282	i40e_shutdown_adminq(hw); /* ignore error */
1283err:
1284	return -1;
1285}
1286
1287static int
1288i40evf_uninit_vf(struct rte_eth_dev *dev)
1289{
1290	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1291	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1292
1293	PMD_INIT_FUNC_TRACE();
1294
1295	if (hw->adapter_stopped == 0)
1296		i40evf_dev_close(dev);
1297	rte_free(vf->vf_res);
1298	vf->vf_res = NULL;
1299	rte_free(vf->aq_resp);
1300	vf->aq_resp = NULL;
1301
1302	return 0;
1303}
1304
1305static void
1306i40evf_handle_pf_event(__rte_unused struct rte_eth_dev *dev,
1307			   uint8_t *msg,
1308			   __rte_unused uint16_t msglen)
1309{
1310	struct i40e_virtchnl_pf_event *pf_msg =
1311			(struct i40e_virtchnl_pf_event *)msg;
1312	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1313
1314	switch (pf_msg->event) {
1315	case I40E_VIRTCHNL_EVENT_RESET_IMPENDING:
1316		PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_RESET_IMPENDING event\n");
1317		_rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET);
1318		break;
1319	case I40E_VIRTCHNL_EVENT_LINK_CHANGE:
1320		PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_LINK_CHANGE event\n");
1321		vf->link_up = pf_msg->event_data.link_event.link_status;
1322		vf->link_speed = pf_msg->event_data.link_event.link_speed;
1323		break;
1324	case I40E_VIRTCHNL_EVENT_PF_DRIVER_CLOSE:
1325		PMD_DRV_LOG(DEBUG, "VIRTCHNL_EVENT_PF_DRIVER_CLOSE event\n");
1326		break;
1327	default:
1328		PMD_DRV_LOG(ERR, " unknown event received %u", pf_msg->event);
1329		break;
1330	}
1331}
1332
1333static void
1334i40evf_handle_aq_msg(struct rte_eth_dev *dev)
1335{
1336	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1337	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1338	struct i40e_arq_event_info info;
1339	struct i40e_virtchnl_msg *v_msg;
1340	uint16_t pending, opcode;
1341	int ret;
1342
1343	info.buf_len = I40E_AQ_BUF_SZ;
1344	if (!vf->aq_resp) {
1345		PMD_DRV_LOG(ERR, "Buffer for adminq resp should not be NULL");
1346		return;
1347	}
1348	info.msg_buf = vf->aq_resp;
1349	v_msg = (struct i40e_virtchnl_msg *)&info.desc;
1350
1351	pending = 1;
1352	while (pending) {
1353		ret = i40e_clean_arq_element(hw, &info, &pending);
1354
1355		if (ret != I40E_SUCCESS) {
1356			PMD_DRV_LOG(INFO, "Failed to read msg from AdminQ,"
1357				    "ret: %d", ret);
1358			break;
1359		}
1360		opcode = rte_le_to_cpu_16(info.desc.opcode);
1361
1362		switch (opcode) {
1363		case i40e_aqc_opc_send_msg_to_vf:
1364			if (v_msg->v_opcode == I40E_VIRTCHNL_OP_EVENT)
1365				/* process event*/
1366				i40evf_handle_pf_event(dev, info.msg_buf,
1367						       info.msg_len);
1368			else {
1369				/* read message and it's expected one */
1370				if (v_msg->v_opcode == vf->pend_cmd) {
1371					vf->cmd_retval = v_msg->v_retval;
1372					/* prevent compiler reordering */
1373					rte_compiler_barrier();
1374					_clear_cmd(vf);
1375				} else
1376					PMD_DRV_LOG(ERR, "command mismatch,"
1377						"expect %u, get %u",
1378						vf->pend_cmd, v_msg->v_opcode);
1379				PMD_DRV_LOG(DEBUG, "adminq response is received,"
1380					     " opcode = %d\n", v_msg->v_opcode);
1381			}
1382			break;
1383		default:
1384			PMD_DRV_LOG(ERR, "Request %u is not supported yet",
1385				    opcode);
1386			break;
1387		}
1388	}
1389}
1390
1391/**
1392 * Interrupt handler triggered by NIC  for handling
1393 * specific interrupt. Only adminq interrupt is processed in VF.
1394 *
1395 * @param handle
1396 *  Pointer to interrupt handle.
1397 * @param param
1398 *  The address of parameter (struct rte_eth_dev *) regsitered before.
1399 *
1400 * @return
1401 *  void
1402 */
1403static void
1404i40evf_dev_interrupt_handler(__rte_unused struct rte_intr_handle *handle,
1405			     void *param)
1406{
1407	struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
1408	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1409	uint32_t icr0;
1410
1411	i40evf_disable_irq0(hw);
1412
1413	/* read out interrupt causes */
1414	icr0 = I40E_READ_REG(hw, I40E_VFINT_ICR01);
1415
1416	/* No interrupt event indicated */
1417	if (!(icr0 & I40E_VFINT_ICR01_INTEVENT_MASK)) {
1418		PMD_DRV_LOG(DEBUG, "No interrupt event, nothing to do\n");
1419		goto done;
1420	}
1421
1422	if (icr0 & I40E_VFINT_ICR01_ADMINQ_MASK) {
1423		PMD_DRV_LOG(DEBUG, "ICR01_ADMINQ is reported\n");
1424		i40evf_handle_aq_msg(dev);
1425	}
1426
1427	/* Link Status Change interrupt */
1428	if (icr0 & I40E_VFINT_ICR01_LINK_STAT_CHANGE_MASK)
1429		PMD_DRV_LOG(DEBUG, "LINK_STAT_CHANGE is reported,"
1430				   " do nothing\n");
1431
1432done:
1433	i40evf_enable_irq0(hw);
1434	rte_intr_enable(&dev->pci_dev->intr_handle);
1435}
1436
1437static int
1438i40evf_dev_init(struct rte_eth_dev *eth_dev)
1439{
1440	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(\
1441			eth_dev->data->dev_private);
1442	struct rte_pci_device *pci_dev = eth_dev->pci_dev;
1443
1444	PMD_INIT_FUNC_TRACE();
1445
1446	/* assign ops func pointer */
1447	eth_dev->dev_ops = &i40evf_eth_dev_ops;
1448	eth_dev->rx_pkt_burst = &i40e_recv_pkts;
1449	eth_dev->tx_pkt_burst = &i40e_xmit_pkts;
1450
1451	/*
1452	 * For secondary processes, we don't initialise any further as primary
1453	 * has already done this work.
1454	 */
1455	if (rte_eal_process_type() != RTE_PROC_PRIMARY){
1456		i40e_set_rx_function(eth_dev);
1457		i40e_set_tx_function(eth_dev);
1458		return 0;
1459	}
1460
1461	rte_eth_copy_pci_info(eth_dev, eth_dev->pci_dev);
1462
1463	hw->vendor_id = eth_dev->pci_dev->id.vendor_id;
1464	hw->device_id = eth_dev->pci_dev->id.device_id;
1465	hw->subsystem_vendor_id = eth_dev->pci_dev->id.subsystem_vendor_id;
1466	hw->subsystem_device_id = eth_dev->pci_dev->id.subsystem_device_id;
1467	hw->bus.device = eth_dev->pci_dev->addr.devid;
1468	hw->bus.func = eth_dev->pci_dev->addr.function;
1469	hw->hw_addr = (void *)eth_dev->pci_dev->mem_resource[0].addr;
1470	hw->adapter_stopped = 0;
1471
1472	if(i40evf_init_vf(eth_dev) != 0) {
1473		PMD_INIT_LOG(ERR, "Init vf failed");
1474		return -1;
1475	}
1476
1477	/* register callback func to eal lib */
1478	rte_intr_callback_register(&pci_dev->intr_handle,
1479		i40evf_dev_interrupt_handler, (void *)eth_dev);
1480
1481	/* enable uio intr after callback register */
1482	rte_intr_enable(&pci_dev->intr_handle);
1483
1484	/* configure and enable device interrupt */
1485	i40evf_enable_irq0(hw);
1486
1487	/* copy mac addr */
1488	eth_dev->data->mac_addrs = rte_zmalloc("i40evf_mac",
1489					ETHER_ADDR_LEN * I40E_NUM_MACADDR_MAX,
1490					0);
1491	if (eth_dev->data->mac_addrs == NULL) {
1492		PMD_INIT_LOG(ERR, "Failed to allocate %d bytes needed to"
1493				" store MAC addresses",
1494				ETHER_ADDR_LEN * I40E_NUM_MACADDR_MAX);
1495		return -ENOMEM;
1496	}
1497	ether_addr_copy((struct ether_addr *)hw->mac.addr,
1498			&eth_dev->data->mac_addrs[0]);
1499
1500	return 0;
1501}
1502
1503static int
1504i40evf_dev_uninit(struct rte_eth_dev *eth_dev)
1505{
1506	PMD_INIT_FUNC_TRACE();
1507
1508	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1509		return -EPERM;
1510
1511	eth_dev->dev_ops = NULL;
1512	eth_dev->rx_pkt_burst = NULL;
1513	eth_dev->tx_pkt_burst = NULL;
1514
1515	if (i40evf_uninit_vf(eth_dev) != 0) {
1516		PMD_INIT_LOG(ERR, "i40evf_uninit_vf failed");
1517		return -1;
1518	}
1519
1520	rte_free(eth_dev->data->mac_addrs);
1521	eth_dev->data->mac_addrs = NULL;
1522
1523	return 0;
1524}
1525/*
1526 * virtual function driver struct
1527 */
1528static struct eth_driver rte_i40evf_pmd = {
1529	.pci_drv = {
1530		.name = "rte_i40evf_pmd",
1531		.id_table = pci_id_i40evf_map,
1532		.drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_DETACHABLE,
1533	},
1534	.eth_dev_init = i40evf_dev_init,
1535	.eth_dev_uninit = i40evf_dev_uninit,
1536	.dev_private_size = sizeof(struct i40e_adapter),
1537};
1538
1539/*
1540 * VF Driver initialization routine.
1541 * Invoked one at EAL init time.
1542 * Register itself as the [Virtual Poll Mode] Driver of PCI Fortville devices.
1543 */
1544static int
1545rte_i40evf_pmd_init(const char *name __rte_unused,
1546		    const char *params __rte_unused)
1547{
1548	PMD_INIT_FUNC_TRACE();
1549
1550	rte_eth_driver_register(&rte_i40evf_pmd);
1551
1552	return 0;
1553}
1554
1555static struct rte_driver rte_i40evf_driver = {
1556	.type = PMD_PDEV,
1557	.init = rte_i40evf_pmd_init,
1558};
1559
1560PMD_REGISTER_DRIVER(rte_i40evf_driver, i40evf);
1561DRIVER_REGISTER_PCI_TABLE(i40evf, pci_id_i40evf_map);
1562
1563static int
1564i40evf_dev_configure(struct rte_eth_dev *dev)
1565{
1566	struct i40e_adapter *ad =
1567		I40E_DEV_PRIVATE_TO_ADAPTER(dev->data->dev_private);
1568	struct rte_eth_conf *conf = &dev->data->dev_conf;
1569	struct i40e_vf *vf;
1570
1571	/* Initialize to TRUE. If any of Rx queues doesn't meet the bulk
1572	 * allocation or vector Rx preconditions we will reset it.
1573	 */
1574	ad->rx_bulk_alloc_allowed = true;
1575	ad->rx_vec_allowed = true;
1576	ad->tx_simple_allowed = true;
1577	ad->tx_vec_allowed = true;
1578
1579	/* For non-DPDK PF drivers, VF has no ability to disable HW
1580	 * CRC strip, and is implicitly enabled by the PF.
1581	 */
1582	if (!conf->rxmode.hw_strip_crc) {
1583		vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1584		if ((vf->version_major == I40E_VIRTCHNL_VERSION_MAJOR) &&
1585		    (vf->version_minor <= I40E_VIRTCHNL_VERSION_MINOR)) {
1586			/* Peer is running non-DPDK PF driver. */
1587			PMD_INIT_LOG(ERR, "VF can't disable HW CRC Strip");
1588			return -EINVAL;
1589		}
1590	}
1591
1592	return i40evf_init_vlan(dev);
1593}
1594
1595static int
1596i40evf_init_vlan(struct rte_eth_dev *dev)
1597{
1598	struct rte_eth_dev_data *data = dev->data;
1599	int ret;
1600
1601	/* Apply vlan offload setting */
1602	i40evf_vlan_offload_set(dev, ETH_VLAN_STRIP_MASK);
1603
1604	/* Apply pvid setting */
1605	ret = i40evf_vlan_pvid_set(dev, data->dev_conf.txmode.pvid,
1606				data->dev_conf.txmode.hw_vlan_insert_pvid);
1607	return ret;
1608}
1609
1610static void
1611i40evf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1612{
1613	bool enable_vlan_strip = 0;
1614	struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
1615	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1616
1617	/* Linux pf host doesn't support vlan offload yet */
1618	if (vf->version_major == I40E_DPDK_VERSION_MAJOR) {
1619		/* Vlan stripping setting */
1620		if (mask & ETH_VLAN_STRIP_MASK) {
1621			/* Enable or disable VLAN stripping */
1622			if (dev_conf->rxmode.hw_vlan_strip)
1623				enable_vlan_strip = 1;
1624			else
1625				enable_vlan_strip = 0;
1626
1627			i40evf_config_vlan_offload(dev, enable_vlan_strip);
1628		}
1629	}
1630}
1631
1632static int
1633i40evf_vlan_pvid_set(struct rte_eth_dev *dev, uint16_t pvid, int on)
1634{
1635	struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
1636	struct i40e_vsi_vlan_pvid_info info;
1637	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1638
1639	memset(&info, 0, sizeof(info));
1640	info.on = on;
1641
1642	/* Linux pf host don't support vlan offload yet */
1643	if (vf->version_major == I40E_DPDK_VERSION_MAJOR) {
1644		if (info.on)
1645			info.config.pvid = pvid;
1646		else {
1647			info.config.reject.tagged =
1648				dev_conf->txmode.hw_vlan_reject_tagged;
1649			info.config.reject.untagged =
1650				dev_conf->txmode.hw_vlan_reject_untagged;
1651		}
1652		return i40evf_config_vlan_pvid(dev, &info);
1653	}
1654
1655	return 0;
1656}
1657
1658static int
1659i40evf_dev_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1660{
1661	struct i40e_rx_queue *rxq;
1662	int err = 0;
1663	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1664
1665	PMD_INIT_FUNC_TRACE();
1666
1667	if (rx_queue_id < dev->data->nb_rx_queues) {
1668		rxq = dev->data->rx_queues[rx_queue_id];
1669
1670		err = i40e_alloc_rx_queue_mbufs(rxq);
1671		if (err) {
1672			PMD_DRV_LOG(ERR, "Failed to allocate RX queue mbuf");
1673			return err;
1674		}
1675
1676		rte_wmb();
1677
1678		/* Init the RX tail register. */
1679		I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
1680		I40EVF_WRITE_FLUSH(hw);
1681
1682		/* Ready to switch the queue on */
1683		err = i40evf_switch_queue(dev, TRUE, rx_queue_id, TRUE);
1684
1685		if (err)
1686			PMD_DRV_LOG(ERR, "Failed to switch RX queue %u on",
1687				    rx_queue_id);
1688		else
1689			dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
1690	}
1691
1692	return err;
1693}
1694
1695static int
1696i40evf_dev_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1697{
1698	struct i40e_rx_queue *rxq;
1699	int err;
1700
1701	if (rx_queue_id < dev->data->nb_rx_queues) {
1702		rxq = dev->data->rx_queues[rx_queue_id];
1703
1704		err = i40evf_switch_queue(dev, TRUE, rx_queue_id, FALSE);
1705
1706		if (err) {
1707			PMD_DRV_LOG(ERR, "Failed to switch RX queue %u off",
1708				    rx_queue_id);
1709			return err;
1710		}
1711
1712		i40e_rx_queue_release_mbufs(rxq);
1713		i40e_reset_rx_queue(rxq);
1714		dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
1715	}
1716
1717	return 0;
1718}
1719
1720static int
1721i40evf_dev_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
1722{
1723	int err = 0;
1724
1725	PMD_INIT_FUNC_TRACE();
1726
1727	if (tx_queue_id < dev->data->nb_tx_queues) {
1728
1729		/* Ready to switch the queue on */
1730		err = i40evf_switch_queue(dev, FALSE, tx_queue_id, TRUE);
1731
1732		if (err)
1733			PMD_DRV_LOG(ERR, "Failed to switch TX queue %u on",
1734				    tx_queue_id);
1735		else
1736			dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
1737	}
1738
1739	return err;
1740}
1741
1742static int
1743i40evf_dev_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
1744{
1745	struct i40e_tx_queue *txq;
1746	int err;
1747
1748	if (tx_queue_id < dev->data->nb_tx_queues) {
1749		txq = dev->data->tx_queues[tx_queue_id];
1750
1751		err = i40evf_switch_queue(dev, FALSE, tx_queue_id, FALSE);
1752
1753		if (err) {
1754			PMD_DRV_LOG(ERR, "Failed to switch TX queue %u off",
1755				    tx_queue_id);
1756			return err;
1757		}
1758
1759		i40e_tx_queue_release_mbufs(txq);
1760		i40e_reset_tx_queue(txq);
1761		dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
1762	}
1763
1764	return 0;
1765}
1766
1767static int
1768i40evf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1769{
1770	int ret;
1771
1772	if (on)
1773		ret = i40evf_add_vlan(dev, vlan_id);
1774	else
1775		ret = i40evf_del_vlan(dev,vlan_id);
1776
1777	return ret;
1778}
1779
1780static int
1781i40evf_rxq_init(struct rte_eth_dev *dev, struct i40e_rx_queue *rxq)
1782{
1783	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1784	struct rte_eth_dev_data *dev_data = dev->data;
1785	struct rte_pktmbuf_pool_private *mbp_priv;
1786	uint16_t buf_size, len;
1787
1788	rxq->qrx_tail = hw->hw_addr + I40E_QRX_TAIL1(rxq->queue_id);
1789	I40E_PCI_REG_WRITE(rxq->qrx_tail, rxq->nb_rx_desc - 1);
1790	I40EVF_WRITE_FLUSH(hw);
1791
1792	/* Calculate the maximum packet length allowed */
1793	mbp_priv = rte_mempool_get_priv(rxq->mp);
1794	buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
1795					RTE_PKTMBUF_HEADROOM);
1796	rxq->hs_mode = i40e_header_split_none;
1797	rxq->rx_hdr_len = 0;
1798	rxq->rx_buf_len = RTE_ALIGN(buf_size, (1 << I40E_RXQ_CTX_DBUFF_SHIFT));
1799	len = rxq->rx_buf_len * I40E_MAX_CHAINED_RX_BUFFERS;
1800	rxq->max_pkt_len = RTE_MIN(len,
1801		dev_data->dev_conf.rxmode.max_rx_pkt_len);
1802
1803	/**
1804	 * Check if the jumbo frame and maximum packet length are set correctly
1805	 */
1806	if (dev_data->dev_conf.rxmode.jumbo_frame == 1) {
1807		if (rxq->max_pkt_len <= ETHER_MAX_LEN ||
1808		    rxq->max_pkt_len > I40E_FRAME_SIZE_MAX) {
1809			PMD_DRV_LOG(ERR, "maximum packet length must be "
1810				"larger than %u and smaller than %u, as jumbo "
1811				"frame is enabled", (uint32_t)ETHER_MAX_LEN,
1812					(uint32_t)I40E_FRAME_SIZE_MAX);
1813			return I40E_ERR_CONFIG;
1814		}
1815	} else {
1816		if (rxq->max_pkt_len < ETHER_MIN_LEN ||
1817		    rxq->max_pkt_len > ETHER_MAX_LEN) {
1818			PMD_DRV_LOG(ERR, "maximum packet length must be "
1819				"larger than %u and smaller than %u, as jumbo "
1820				"frame is disabled", (uint32_t)ETHER_MIN_LEN,
1821						(uint32_t)ETHER_MAX_LEN);
1822			return I40E_ERR_CONFIG;
1823		}
1824	}
1825
1826	if (dev_data->dev_conf.rxmode.enable_scatter ||
1827	    (rxq->max_pkt_len + 2 * I40E_VLAN_TAG_SIZE) > buf_size) {
1828		dev_data->scattered_rx = 1;
1829	}
1830
1831	return 0;
1832}
1833
1834static int
1835i40evf_rx_init(struct rte_eth_dev *dev)
1836{
1837	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1838	uint16_t i;
1839	int ret = I40E_SUCCESS;
1840	struct i40e_rx_queue **rxq =
1841		(struct i40e_rx_queue **)dev->data->rx_queues;
1842
1843	i40evf_config_rss(vf);
1844	for (i = 0; i < dev->data->nb_rx_queues; i++) {
1845		if (!rxq[i] || !rxq[i]->q_set)
1846			continue;
1847		ret = i40evf_rxq_init(dev, rxq[i]);
1848		if (ret != I40E_SUCCESS)
1849			break;
1850	}
1851	if (ret == I40E_SUCCESS)
1852		i40e_set_rx_function(dev);
1853
1854	return ret;
1855}
1856
1857static void
1858i40evf_tx_init(struct rte_eth_dev *dev)
1859{
1860	uint16_t i;
1861	struct i40e_tx_queue **txq =
1862		(struct i40e_tx_queue **)dev->data->tx_queues;
1863	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1864
1865	for (i = 0; i < dev->data->nb_tx_queues; i++)
1866		txq[i]->qtx_tail = hw->hw_addr + I40E_QTX_TAIL1(i);
1867
1868	i40e_set_tx_function(dev);
1869}
1870
1871static inline void
1872i40evf_enable_queues_intr(struct rte_eth_dev *dev)
1873{
1874	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1875	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1876	struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1877
1878	if (!rte_intr_allow_others(intr_handle)) {
1879		I40E_WRITE_REG(hw,
1880			       I40E_VFINT_DYN_CTL01,
1881			       I40E_VFINT_DYN_CTL01_INTENA_MASK |
1882			       I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
1883			       I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
1884		I40EVF_WRITE_FLUSH(hw);
1885		return;
1886	}
1887
1888	if (vf->version_major == I40E_DPDK_VERSION_MAJOR)
1889		/* To support DPDK PF host */
1890		I40E_WRITE_REG(hw,
1891			I40E_VFINT_DYN_CTLN1(I40EVF_VSI_DEFAULT_MSIX_INTR - 1),
1892			I40E_VFINT_DYN_CTLN1_INTENA_MASK |
1893			I40E_VFINT_DYN_CTLN_CLEARPBA_MASK);
1894	/* If host driver is kernel driver, do nothing.
1895	 * Interrupt 0 is used for rx packets, but don't set
1896	 * I40E_VFINT_DYN_CTL01,
1897	 * because it is already done in i40evf_enable_irq0.
1898	 */
1899
1900	I40EVF_WRITE_FLUSH(hw);
1901}
1902
1903static inline void
1904i40evf_disable_queues_intr(struct rte_eth_dev *dev)
1905{
1906	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1907	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1908	struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1909
1910	if (!rte_intr_allow_others(intr_handle)) {
1911		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
1912			       I40E_VFINT_DYN_CTL01_ITR_INDX_MASK);
1913		I40EVF_WRITE_FLUSH(hw);
1914		return;
1915	}
1916
1917	if (vf->version_major == I40E_DPDK_VERSION_MAJOR)
1918		I40E_WRITE_REG(hw,
1919			       I40E_VFINT_DYN_CTLN1(I40EVF_VSI_DEFAULT_MSIX_INTR
1920						    - 1),
1921			       0);
1922	/* If host driver is kernel driver, do nothing.
1923	 * Interrupt 0 is used for rx packets, but don't zero
1924	 * I40E_VFINT_DYN_CTL01,
1925	 * because interrupt 0 is also used for adminq processing.
1926	 */
1927
1928	I40EVF_WRITE_FLUSH(hw);
1929}
1930
1931static int
1932i40evf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
1933{
1934	struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1935	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1936	uint16_t interval =
1937		i40e_calc_itr_interval(RTE_LIBRTE_I40E_ITR_INTERVAL);
1938	uint16_t msix_intr;
1939
1940	msix_intr = intr_handle->intr_vec[queue_id];
1941	if (msix_intr == I40E_MISC_VEC_ID)
1942		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01,
1943			       I40E_VFINT_DYN_CTL01_INTENA_MASK |
1944			       I40E_VFINT_DYN_CTL01_CLEARPBA_MASK |
1945			       (0 << I40E_VFINT_DYN_CTL01_ITR_INDX_SHIFT) |
1946			       (interval <<
1947				I40E_VFINT_DYN_CTL01_INTERVAL_SHIFT));
1948	else
1949		I40E_WRITE_REG(hw,
1950			       I40E_VFINT_DYN_CTLN1(msix_intr -
1951						    I40E_RX_VEC_START),
1952			       I40E_VFINT_DYN_CTLN1_INTENA_MASK |
1953			       I40E_VFINT_DYN_CTLN1_CLEARPBA_MASK |
1954			       (0 << I40E_VFINT_DYN_CTLN1_ITR_INDX_SHIFT) |
1955			       (interval <<
1956				I40E_VFINT_DYN_CTLN1_INTERVAL_SHIFT));
1957
1958	I40EVF_WRITE_FLUSH(hw);
1959
1960	rte_intr_enable(&dev->pci_dev->intr_handle);
1961
1962	return 0;
1963}
1964
1965static int
1966i40evf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
1967{
1968	struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
1969	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1970	uint16_t msix_intr;
1971
1972	msix_intr = intr_handle->intr_vec[queue_id];
1973	if (msix_intr == I40E_MISC_VEC_ID)
1974		I40E_WRITE_REG(hw, I40E_VFINT_DYN_CTL01, 0);
1975	else
1976		I40E_WRITE_REG(hw,
1977			       I40E_VFINT_DYN_CTLN1(msix_intr -
1978						    I40E_RX_VEC_START),
1979			       0);
1980
1981	I40EVF_WRITE_FLUSH(hw);
1982
1983	return 0;
1984}
1985
1986static void
1987i40evf_add_del_all_mac_addr(struct rte_eth_dev *dev, bool add)
1988{
1989	struct i40e_virtchnl_ether_addr_list *list;
1990	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
1991	int err, i, j;
1992	int next_begin = 0;
1993	int begin = 0;
1994	uint32_t len;
1995	struct ether_addr *addr;
1996	struct vf_cmd_info args;
1997
1998	do {
1999		j = 0;
2000		len = sizeof(struct i40e_virtchnl_ether_addr_list);
2001		for (i = begin; i < I40E_NUM_MACADDR_MAX; i++, next_begin++) {
2002			if (is_zero_ether_addr(&dev->data->mac_addrs[i]))
2003				continue;
2004			len += sizeof(struct i40e_virtchnl_ether_addr);
2005			if (len >= I40E_AQ_BUF_SZ) {
2006				next_begin = i + 1;
2007				break;
2008			}
2009		}
2010
2011		list = rte_zmalloc("i40evf_del_mac_buffer", len, 0);
2012
2013		for (i = begin; i < next_begin; i++) {
2014			addr = &dev->data->mac_addrs[i];
2015			if (is_zero_ether_addr(addr))
2016				continue;
2017			(void)rte_memcpy(list->list[j].addr, addr->addr_bytes,
2018					 sizeof(addr->addr_bytes));
2019			PMD_DRV_LOG(DEBUG, "add/rm mac:%x:%x:%x:%x:%x:%x",
2020				    addr->addr_bytes[0], addr->addr_bytes[1],
2021				    addr->addr_bytes[2], addr->addr_bytes[3],
2022				    addr->addr_bytes[4], addr->addr_bytes[5]);
2023			j++;
2024		}
2025		list->vsi_id = vf->vsi_res->vsi_id;
2026		list->num_elements = j;
2027		args.ops = add ? I40E_VIRTCHNL_OP_ADD_ETHER_ADDRESS :
2028			   I40E_VIRTCHNL_OP_DEL_ETHER_ADDRESS;
2029		args.in_args = (uint8_t *)list;
2030		args.in_args_size = len;
2031		args.out_buffer = vf->aq_resp;
2032		args.out_size = I40E_AQ_BUF_SZ;
2033		err = i40evf_execute_vf_cmd(dev, &args);
2034		if (err)
2035			PMD_DRV_LOG(ERR, "fail to execute command %s",
2036				    add ? "OP_ADD_ETHER_ADDRESS" :
2037				    "OP_DEL_ETHER_ADDRESS");
2038		rte_free(list);
2039		begin = next_begin;
2040	} while (begin < I40E_NUM_MACADDR_MAX);
2041}
2042
2043static int
2044i40evf_dev_start(struct rte_eth_dev *dev)
2045{
2046	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2047	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2048	struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
2049	uint32_t intr_vector = 0;
2050
2051	PMD_INIT_FUNC_TRACE();
2052
2053	hw->adapter_stopped = 0;
2054
2055	vf->max_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
2056	vf->num_queue_pairs = RTE_MAX(dev->data->nb_rx_queues,
2057					dev->data->nb_tx_queues);
2058
2059	/* check and configure queue intr-vector mapping */
2060	if (dev->data->dev_conf.intr_conf.rxq != 0) {
2061		intr_vector = dev->data->nb_rx_queues;
2062		if (rte_intr_efd_enable(intr_handle, intr_vector))
2063			return -1;
2064	}
2065
2066	if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
2067		intr_handle->intr_vec =
2068			rte_zmalloc("intr_vec",
2069				    dev->data->nb_rx_queues * sizeof(int), 0);
2070		if (!intr_handle->intr_vec) {
2071			PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2072				     " intr_vec\n", dev->data->nb_rx_queues);
2073			return -ENOMEM;
2074		}
2075	}
2076
2077	if (i40evf_rx_init(dev) != 0){
2078		PMD_DRV_LOG(ERR, "failed to do RX init");
2079		return -1;
2080	}
2081
2082	i40evf_tx_init(dev);
2083
2084	if (i40evf_configure_queues(dev) != 0) {
2085		PMD_DRV_LOG(ERR, "configure queues failed");
2086		goto err_queue;
2087	}
2088	if (i40evf_config_irq_map(dev)) {
2089		PMD_DRV_LOG(ERR, "config_irq_map failed");
2090		goto err_queue;
2091	}
2092
2093	/* Set all mac addrs */
2094	i40evf_add_del_all_mac_addr(dev, TRUE);
2095
2096	if (i40evf_start_queues(dev) != 0) {
2097		PMD_DRV_LOG(ERR, "enable queues failed");
2098		goto err_mac;
2099	}
2100
2101	i40evf_enable_queues_intr(dev);
2102	return 0;
2103
2104err_mac:
2105	i40evf_add_del_all_mac_addr(dev, FALSE);
2106err_queue:
2107	return -1;
2108}
2109
2110static void
2111i40evf_dev_stop(struct rte_eth_dev *dev)
2112{
2113	struct rte_intr_handle *intr_handle = &dev->pci_dev->intr_handle;
2114
2115	PMD_INIT_FUNC_TRACE();
2116
2117	i40evf_stop_queues(dev);
2118	i40evf_disable_queues_intr(dev);
2119	i40e_dev_clear_queues(dev);
2120
2121	/* Clean datapath event and queue/vec mapping */
2122	rte_intr_efd_disable(intr_handle);
2123	if (intr_handle->intr_vec) {
2124		rte_free(intr_handle->intr_vec);
2125		intr_handle->intr_vec = NULL;
2126	}
2127	/* remove all mac addrs */
2128	i40evf_add_del_all_mac_addr(dev, FALSE);
2129
2130}
2131
2132static int
2133i40evf_dev_link_update(struct rte_eth_dev *dev,
2134		       __rte_unused int wait_to_complete)
2135{
2136	struct rte_eth_link new_link;
2137	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2138	/*
2139	 * DPDK pf host provide interfacet to acquire link status
2140	 * while Linux driver does not
2141	 */
2142
2143	/* Linux driver PF host */
2144	switch (vf->link_speed) {
2145	case I40E_LINK_SPEED_100MB:
2146		new_link.link_speed = ETH_SPEED_NUM_100M;
2147		break;
2148	case I40E_LINK_SPEED_1GB:
2149		new_link.link_speed = ETH_SPEED_NUM_1G;
2150		break;
2151	case I40E_LINK_SPEED_10GB:
2152		new_link.link_speed = ETH_SPEED_NUM_10G;
2153		break;
2154	case I40E_LINK_SPEED_20GB:
2155		new_link.link_speed = ETH_SPEED_NUM_20G;
2156		break;
2157	case I40E_LINK_SPEED_40GB:
2158		new_link.link_speed = ETH_SPEED_NUM_40G;
2159		break;
2160	default:
2161		new_link.link_speed = ETH_SPEED_NUM_100M;
2162		break;
2163	}
2164	/* full duplex only */
2165	new_link.link_duplex = ETH_LINK_FULL_DUPLEX;
2166	new_link.link_status = vf->link_up ? ETH_LINK_UP :
2167					     ETH_LINK_DOWN;
2168
2169	i40evf_dev_atomic_write_link_status(dev, &new_link);
2170
2171	return 0;
2172}
2173
2174static void
2175i40evf_dev_promiscuous_enable(struct rte_eth_dev *dev)
2176{
2177	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2178	int ret;
2179
2180	/* If enabled, just return */
2181	if (vf->promisc_unicast_enabled)
2182		return;
2183
2184	ret = i40evf_config_promisc(dev, 1, vf->promisc_multicast_enabled);
2185	if (ret == 0)
2186		vf->promisc_unicast_enabled = TRUE;
2187}
2188
2189static void
2190i40evf_dev_promiscuous_disable(struct rte_eth_dev *dev)
2191{
2192	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2193	int ret;
2194
2195	/* If disabled, just return */
2196	if (!vf->promisc_unicast_enabled)
2197		return;
2198
2199	ret = i40evf_config_promisc(dev, 0, vf->promisc_multicast_enabled);
2200	if (ret == 0)
2201		vf->promisc_unicast_enabled = FALSE;
2202}
2203
2204static void
2205i40evf_dev_allmulticast_enable(struct rte_eth_dev *dev)
2206{
2207	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2208	int ret;
2209
2210	/* If enabled, just return */
2211	if (vf->promisc_multicast_enabled)
2212		return;
2213
2214	ret = i40evf_config_promisc(dev, vf->promisc_unicast_enabled, 1);
2215	if (ret == 0)
2216		vf->promisc_multicast_enabled = TRUE;
2217}
2218
2219static void
2220i40evf_dev_allmulticast_disable(struct rte_eth_dev *dev)
2221{
2222	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2223	int ret;
2224
2225	/* If enabled, just return */
2226	if (!vf->promisc_multicast_enabled)
2227		return;
2228
2229	ret = i40evf_config_promisc(dev, vf->promisc_unicast_enabled, 0);
2230	if (ret == 0)
2231		vf->promisc_multicast_enabled = FALSE;
2232}
2233
2234static void
2235i40evf_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
2236{
2237	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2238
2239	memset(dev_info, 0, sizeof(*dev_info));
2240	dev_info->max_rx_queues = vf->vsi_res->num_queue_pairs;
2241	dev_info->max_tx_queues = vf->vsi_res->num_queue_pairs;
2242	dev_info->min_rx_bufsize = I40E_BUF_SIZE_MIN;
2243	dev_info->max_rx_pktlen = I40E_FRAME_SIZE_MAX;
2244	dev_info->hash_key_size = (I40E_VFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t);
2245	dev_info->reta_size = ETH_RSS_RETA_SIZE_64;
2246	dev_info->flow_type_rss_offloads = I40E_RSS_OFFLOAD_ALL;
2247	dev_info->max_mac_addrs = I40E_NUM_MACADDR_MAX;
2248	dev_info->rx_offload_capa =
2249		DEV_RX_OFFLOAD_VLAN_STRIP |
2250		DEV_RX_OFFLOAD_QINQ_STRIP |
2251		DEV_RX_OFFLOAD_IPV4_CKSUM |
2252		DEV_RX_OFFLOAD_UDP_CKSUM |
2253		DEV_RX_OFFLOAD_TCP_CKSUM;
2254	dev_info->tx_offload_capa =
2255		DEV_TX_OFFLOAD_VLAN_INSERT |
2256		DEV_TX_OFFLOAD_QINQ_INSERT |
2257		DEV_TX_OFFLOAD_IPV4_CKSUM |
2258		DEV_TX_OFFLOAD_UDP_CKSUM |
2259		DEV_TX_OFFLOAD_TCP_CKSUM |
2260		DEV_TX_OFFLOAD_SCTP_CKSUM;
2261
2262	dev_info->default_rxconf = (struct rte_eth_rxconf) {
2263		.rx_thresh = {
2264			.pthresh = I40E_DEFAULT_RX_PTHRESH,
2265			.hthresh = I40E_DEFAULT_RX_HTHRESH,
2266			.wthresh = I40E_DEFAULT_RX_WTHRESH,
2267		},
2268		.rx_free_thresh = I40E_DEFAULT_RX_FREE_THRESH,
2269		.rx_drop_en = 0,
2270	};
2271
2272	dev_info->default_txconf = (struct rte_eth_txconf) {
2273		.tx_thresh = {
2274			.pthresh = I40E_DEFAULT_TX_PTHRESH,
2275			.hthresh = I40E_DEFAULT_TX_HTHRESH,
2276			.wthresh = I40E_DEFAULT_TX_WTHRESH,
2277		},
2278		.tx_free_thresh = I40E_DEFAULT_TX_FREE_THRESH,
2279		.tx_rs_thresh = I40E_DEFAULT_TX_RSBIT_THRESH,
2280		.txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
2281				ETH_TXQ_FLAGS_NOOFFLOADS,
2282	};
2283
2284	dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
2285		.nb_max = I40E_MAX_RING_DESC,
2286		.nb_min = I40E_MIN_RING_DESC,
2287		.nb_align = I40E_ALIGN_RING_DESC,
2288	};
2289
2290	dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
2291		.nb_max = I40E_MAX_RING_DESC,
2292		.nb_min = I40E_MIN_RING_DESC,
2293		.nb_align = I40E_ALIGN_RING_DESC,
2294	};
2295}
2296
2297static void
2298i40evf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
2299{
2300	if (i40evf_get_statics(dev, stats))
2301		PMD_DRV_LOG(ERR, "Get statics failed");
2302}
2303
2304static void
2305i40evf_dev_close(struct rte_eth_dev *dev)
2306{
2307	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2308	struct rte_pci_device *pci_dev = dev->pci_dev;
2309
2310	i40evf_dev_stop(dev);
2311	hw->adapter_stopped = 1;
2312	i40e_dev_free_queues(dev);
2313	i40evf_reset_vf(hw);
2314	i40e_shutdown_adminq(hw);
2315	/* disable uio intr before callback unregister */
2316	rte_intr_disable(&pci_dev->intr_handle);
2317
2318	/* unregister callback func from eal lib */
2319	rte_intr_callback_unregister(&pci_dev->intr_handle,
2320		i40evf_dev_interrupt_handler, (void *)dev);
2321	i40evf_disable_irq0(hw);
2322}
2323
2324static int
2325i40evf_get_rss_lut(struct i40e_vsi *vsi, uint8_t *lut, uint16_t lut_size)
2326{
2327	struct i40e_vf *vf = I40E_VSI_TO_VF(vsi);
2328	struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2329	int ret;
2330
2331	if (!lut)
2332		return -EINVAL;
2333
2334	if (vf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
2335		ret = i40e_aq_get_rss_lut(hw, vsi->vsi_id, FALSE,
2336					  lut, lut_size);
2337		if (ret) {
2338			PMD_DRV_LOG(ERR, "Failed to get RSS lookup table");
2339			return ret;
2340		}
2341	} else {
2342		uint32_t *lut_dw = (uint32_t *)lut;
2343		uint16_t i, lut_size_dw = lut_size / 4;
2344
2345		for (i = 0; i < lut_size_dw; i++)
2346			lut_dw[i] = I40E_READ_REG(hw, I40E_VFQF_HLUT(i));
2347	}
2348
2349	return 0;
2350}
2351
2352static int
2353i40evf_set_rss_lut(struct i40e_vsi *vsi, uint8_t *lut, uint16_t lut_size)
2354{
2355	struct i40e_vf *vf;
2356	struct i40e_hw *hw;
2357	int ret;
2358
2359	if (!vsi || !lut)
2360		return -EINVAL;
2361
2362	vf = I40E_VSI_TO_VF(vsi);
2363	hw = I40E_VSI_TO_HW(vsi);
2364
2365	if (vf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
2366		ret = i40e_aq_set_rss_lut(hw, vsi->vsi_id, FALSE,
2367					  lut, lut_size);
2368		if (ret) {
2369			PMD_DRV_LOG(ERR, "Failed to set RSS lookup table");
2370			return ret;
2371		}
2372	} else {
2373		uint32_t *lut_dw = (uint32_t *)lut;
2374		uint16_t i, lut_size_dw = lut_size / 4;
2375
2376		for (i = 0; i < lut_size_dw; i++)
2377			I40E_WRITE_REG(hw, I40E_VFQF_HLUT(i), lut_dw[i]);
2378		I40EVF_WRITE_FLUSH(hw);
2379	}
2380
2381	return 0;
2382}
2383
2384static int
2385i40evf_dev_rss_reta_update(struct rte_eth_dev *dev,
2386			   struct rte_eth_rss_reta_entry64 *reta_conf,
2387			   uint16_t reta_size)
2388{
2389	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2390	uint8_t *lut;
2391	uint16_t i, idx, shift;
2392	int ret;
2393
2394	if (reta_size != ETH_RSS_RETA_SIZE_64) {
2395		PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2396			"(%d) doesn't match the number of hardware can "
2397			"support (%d)\n", reta_size, ETH_RSS_RETA_SIZE_64);
2398		return -EINVAL;
2399	}
2400
2401	lut = rte_zmalloc("i40e_rss_lut", reta_size, 0);
2402	if (!lut) {
2403		PMD_DRV_LOG(ERR, "No memory can be allocated");
2404		return -ENOMEM;
2405	}
2406	ret = i40evf_get_rss_lut(&vf->vsi, lut, reta_size);
2407	if (ret)
2408		goto out;
2409	for (i = 0; i < reta_size; i++) {
2410		idx = i / RTE_RETA_GROUP_SIZE;
2411		shift = i % RTE_RETA_GROUP_SIZE;
2412		if (reta_conf[idx].mask & (1ULL << shift))
2413			lut[i] = reta_conf[idx].reta[shift];
2414	}
2415	ret = i40evf_set_rss_lut(&vf->vsi, lut, reta_size);
2416
2417out:
2418	rte_free(lut);
2419
2420	return ret;
2421}
2422
2423static int
2424i40evf_dev_rss_reta_query(struct rte_eth_dev *dev,
2425			  struct rte_eth_rss_reta_entry64 *reta_conf,
2426			  uint16_t reta_size)
2427{
2428	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2429	uint16_t i, idx, shift;
2430	uint8_t *lut;
2431	int ret;
2432
2433	if (reta_size != ETH_RSS_RETA_SIZE_64) {
2434		PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
2435			"(%d) doesn't match the number of hardware can "
2436			"support (%d)\n", reta_size, ETH_RSS_RETA_SIZE_64);
2437		return -EINVAL;
2438	}
2439
2440	lut = rte_zmalloc("i40e_rss_lut", reta_size, 0);
2441	if (!lut) {
2442		PMD_DRV_LOG(ERR, "No memory can be allocated");
2443		return -ENOMEM;
2444	}
2445
2446	ret = i40evf_get_rss_lut(&vf->vsi, lut, reta_size);
2447	if (ret)
2448		goto out;
2449	for (i = 0; i < reta_size; i++) {
2450		idx = i / RTE_RETA_GROUP_SIZE;
2451		shift = i % RTE_RETA_GROUP_SIZE;
2452		if (reta_conf[idx].mask & (1ULL << shift))
2453			reta_conf[idx].reta[shift] = lut[i];
2454	}
2455
2456out:
2457	rte_free(lut);
2458
2459	return ret;
2460}
2461
2462static int
2463i40evf_set_rss_key(struct i40e_vsi *vsi, uint8_t *key, uint8_t key_len)
2464{
2465	struct i40e_vf *vf = I40E_VSI_TO_VF(vsi);
2466	struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2467	int ret = 0;
2468
2469	if (!key || key_len == 0) {
2470		PMD_DRV_LOG(DEBUG, "No key to be configured");
2471		return 0;
2472	} else if (key_len != (I40E_VFQF_HKEY_MAX_INDEX + 1) *
2473		sizeof(uint32_t)) {
2474		PMD_DRV_LOG(ERR, "Invalid key length %u", key_len);
2475		return -EINVAL;
2476	}
2477
2478	if (vf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
2479		struct i40e_aqc_get_set_rss_key_data *key_dw =
2480			(struct i40e_aqc_get_set_rss_key_data *)key;
2481
2482		ret = i40e_aq_set_rss_key(hw, vsi->vsi_id, key_dw);
2483		if (ret)
2484			PMD_INIT_LOG(ERR, "Failed to configure RSS key "
2485				     "via AQ");
2486	} else {
2487		uint32_t *hash_key = (uint32_t *)key;
2488		uint16_t i;
2489
2490		for (i = 0; i <= I40E_VFQF_HKEY_MAX_INDEX; i++)
2491			i40e_write_rx_ctl(hw, I40E_VFQF_HKEY(i), hash_key[i]);
2492		I40EVF_WRITE_FLUSH(hw);
2493	}
2494
2495	return ret;
2496}
2497
2498static int
2499i40evf_get_rss_key(struct i40e_vsi *vsi, uint8_t *key, uint8_t *key_len)
2500{
2501	struct i40e_vf *vf = I40E_VSI_TO_VF(vsi);
2502	struct i40e_hw *hw = I40E_VSI_TO_HW(vsi);
2503	int ret;
2504
2505	if (!key || !key_len)
2506		return -EINVAL;
2507
2508	if (vf->flags & I40E_FLAG_RSS_AQ_CAPABLE) {
2509		ret = i40e_aq_get_rss_key(hw, vsi->vsi_id,
2510			(struct i40e_aqc_get_set_rss_key_data *)key);
2511		if (ret) {
2512			PMD_INIT_LOG(ERR, "Failed to get RSS key via AQ");
2513			return ret;
2514		}
2515	} else {
2516		uint32_t *key_dw = (uint32_t *)key;
2517		uint16_t i;
2518
2519		for (i = 0; i <= I40E_VFQF_HKEY_MAX_INDEX; i++)
2520			key_dw[i] = i40e_read_rx_ctl(hw, I40E_VFQF_HKEY(i));
2521	}
2522	*key_len = (I40E_VFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t);
2523
2524	return 0;
2525}
2526
2527static int
2528i40evf_hw_rss_hash_set(struct i40e_vf *vf, struct rte_eth_rss_conf *rss_conf)
2529{
2530	struct i40e_hw *hw = I40E_VF_TO_HW(vf);
2531	uint64_t rss_hf, hena;
2532	int ret;
2533
2534	ret = i40evf_set_rss_key(&vf->vsi, rss_conf->rss_key,
2535				 rss_conf->rss_key_len);
2536	if (ret)
2537		return ret;
2538
2539	rss_hf = rss_conf->rss_hf;
2540	hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(0));
2541	hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(1))) << 32;
2542	hena &= ~I40E_RSS_HENA_ALL;
2543	hena |= i40e_config_hena(rss_hf);
2544	i40e_write_rx_ctl(hw, I40E_VFQF_HENA(0), (uint32_t)hena);
2545	i40e_write_rx_ctl(hw, I40E_VFQF_HENA(1), (uint32_t)(hena >> 32));
2546	I40EVF_WRITE_FLUSH(hw);
2547
2548	return 0;
2549}
2550
2551static void
2552i40evf_disable_rss(struct i40e_vf *vf)
2553{
2554	struct i40e_hw *hw = I40E_VF_TO_HW(vf);
2555	uint64_t hena;
2556
2557	hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(0));
2558	hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(1))) << 32;
2559	hena &= ~I40E_RSS_HENA_ALL;
2560	i40e_write_rx_ctl(hw, I40E_VFQF_HENA(0), (uint32_t)hena);
2561	i40e_write_rx_ctl(hw, I40E_VFQF_HENA(1), (uint32_t)(hena >> 32));
2562	I40EVF_WRITE_FLUSH(hw);
2563}
2564
2565static int
2566i40evf_config_rss(struct i40e_vf *vf)
2567{
2568	struct i40e_hw *hw = I40E_VF_TO_HW(vf);
2569	struct rte_eth_rss_conf rss_conf;
2570	uint32_t i, j, lut = 0, nb_q = (I40E_VFQF_HLUT_MAX_INDEX + 1) * 4;
2571	uint16_t num;
2572
2573	if (vf->dev_data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
2574		i40evf_disable_rss(vf);
2575		PMD_DRV_LOG(DEBUG, "RSS not configured\n");
2576		return 0;
2577	}
2578
2579	num = RTE_MIN(vf->dev_data->nb_rx_queues, I40E_MAX_QP_NUM_PER_VF);
2580	/* Fill out the look up table */
2581	for (i = 0, j = 0; i < nb_q; i++, j++) {
2582		if (j >= num)
2583			j = 0;
2584		lut = (lut << 8) | j;
2585		if ((i & 3) == 3)
2586			I40E_WRITE_REG(hw, I40E_VFQF_HLUT(i >> 2), lut);
2587	}
2588
2589	rss_conf = vf->dev_data->dev_conf.rx_adv_conf.rss_conf;
2590	if ((rss_conf.rss_hf & I40E_RSS_OFFLOAD_ALL) == 0) {
2591		i40evf_disable_rss(vf);
2592		PMD_DRV_LOG(DEBUG, "No hash flag is set\n");
2593		return 0;
2594	}
2595
2596	if (rss_conf.rss_key == NULL || rss_conf.rss_key_len <
2597		(I40E_VFQF_HKEY_MAX_INDEX + 1) * sizeof(uint32_t)) {
2598		/* Calculate the default hash key */
2599		for (i = 0; i <= I40E_VFQF_HKEY_MAX_INDEX; i++)
2600			rss_key_default[i] = (uint32_t)rte_rand();
2601		rss_conf.rss_key = (uint8_t *)rss_key_default;
2602		rss_conf.rss_key_len = (I40E_VFQF_HKEY_MAX_INDEX + 1) *
2603			sizeof(uint32_t);
2604	}
2605
2606	return i40evf_hw_rss_hash_set(vf, &rss_conf);
2607}
2608
2609static int
2610i40evf_dev_rss_hash_update(struct rte_eth_dev *dev,
2611			   struct rte_eth_rss_conf *rss_conf)
2612{
2613	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2614	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2615	uint64_t rss_hf = rss_conf->rss_hf & I40E_RSS_OFFLOAD_ALL;
2616	uint64_t hena;
2617
2618	hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(0));
2619	hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(1))) << 32;
2620	if (!(hena & I40E_RSS_HENA_ALL)) { /* RSS disabled */
2621		if (rss_hf != 0) /* Enable RSS */
2622			return -EINVAL;
2623		return 0;
2624	}
2625
2626	/* RSS enabled */
2627	if (rss_hf == 0) /* Disable RSS */
2628		return -EINVAL;
2629
2630	return i40evf_hw_rss_hash_set(vf, rss_conf);
2631}
2632
2633static int
2634i40evf_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
2635			     struct rte_eth_rss_conf *rss_conf)
2636{
2637	struct i40e_vf *vf = I40EVF_DEV_PRIVATE_TO_VF(dev->data->dev_private);
2638	struct i40e_hw *hw = I40E_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2639	uint64_t hena;
2640
2641	i40evf_get_rss_key(&vf->vsi, rss_conf->rss_key,
2642			   &rss_conf->rss_key_len);
2643
2644	hena = (uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(0));
2645	hena |= ((uint64_t)i40e_read_rx_ctl(hw, I40E_VFQF_HENA(1))) << 32;
2646	rss_conf->rss_hf = i40e_parse_hena(hena);
2647
2648	return 0;
2649}
2650