ixgbe_ethdev.c revision 93f15e30
1/*-
2 *   BSD LICENSE
3 *
4 *   Copyright(c) 2010-2017 Intel Corporation. All rights reserved.
5 *   All rights reserved.
6 *
7 *   Redistribution and use in source and binary forms, with or without
8 *   modification, are permitted provided that the following conditions
9 *   are met:
10 *
11 *     * Redistributions of source code must retain the above copyright
12 *       notice, this list of conditions and the following disclaimer.
13 *     * Redistributions in binary form must reproduce the above copyright
14 *       notice, this list of conditions and the following disclaimer in
15 *       the documentation and/or other materials provided with the
16 *       distribution.
17 *     * Neither the name of Intel Corporation nor the names of its
18 *       contributors may be used to endorse or promote products derived
19 *       from this software without specific prior written permission.
20 *
21 *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 */
33
34#include <sys/queue.h>
35#include <stdio.h>
36#include <errno.h>
37#include <stdint.h>
38#include <string.h>
39#include <unistd.h>
40#include <stdarg.h>
41#include <inttypes.h>
42#include <netinet/in.h>
43#include <rte_byteorder.h>
44#include <rte_common.h>
45#include <rte_cycles.h>
46
47#include <rte_interrupts.h>
48#include <rte_log.h>
49#include <rte_debug.h>
50#include <rte_pci.h>
51#include <rte_atomic.h>
52#include <rte_branch_prediction.h>
53#include <rte_memory.h>
54#include <rte_memzone.h>
55#include <rte_eal.h>
56#include <rte_alarm.h>
57#include <rte_ether.h>
58#include <rte_ethdev.h>
59#include <rte_atomic.h>
60#include <rte_malloc.h>
61#include <rte_random.h>
62#include <rte_dev.h>
63#include <rte_hash_crc.h>
64
65#include "ixgbe_logs.h"
66#include "base/ixgbe_api.h"
67#include "base/ixgbe_vf.h"
68#include "base/ixgbe_common.h"
69#include "ixgbe_ethdev.h"
70#include "ixgbe_bypass.h"
71#include "ixgbe_rxtx.h"
72#include "base/ixgbe_type.h"
73#include "base/ixgbe_phy.h"
74#include "ixgbe_regs.h"
75
76#include "rte_pmd_ixgbe.h"
77
78/*
79 * High threshold controlling when to start sending XOFF frames. Must be at
80 * least 8 bytes less than receive packet buffer size. This value is in units
81 * of 1024 bytes.
82 */
83#define IXGBE_FC_HI    0x80
84
85/*
86 * Low threshold controlling when to start sending XON frames. This value is
87 * in units of 1024 bytes.
88 */
89#define IXGBE_FC_LO    0x40
90
91/* Default minimum inter-interrupt interval for EITR configuration */
92#define IXGBE_MIN_INTER_INTERRUPT_INTERVAL_DEFAULT    0x79E
93
94/* Timer value included in XOFF frames. */
95#define IXGBE_FC_PAUSE 0x680
96
97#define IXGBE_LINK_DOWN_CHECK_TIMEOUT 4000 /* ms */
98#define IXGBE_LINK_UP_CHECK_TIMEOUT   1000 /* ms */
99#define IXGBE_VMDQ_NUM_UC_MAC         4096 /* Maximum nb. of UC MAC addr. */
100
101#define IXGBE_MMW_SIZE_DEFAULT        0x4
102#define IXGBE_MMW_SIZE_JUMBO_FRAME    0x14
103#define IXGBE_MAX_RING_DESC           4096 /* replicate define from rxtx */
104
105/*
106 *  Default values for RX/TX configuration
107 */
108#define IXGBE_DEFAULT_RX_FREE_THRESH  32
109#define IXGBE_DEFAULT_RX_PTHRESH      8
110#define IXGBE_DEFAULT_RX_HTHRESH      8
111#define IXGBE_DEFAULT_RX_WTHRESH      0
112
113#define IXGBE_DEFAULT_TX_FREE_THRESH  32
114#define IXGBE_DEFAULT_TX_PTHRESH      32
115#define IXGBE_DEFAULT_TX_HTHRESH      0
116#define IXGBE_DEFAULT_TX_WTHRESH      0
117#define IXGBE_DEFAULT_TX_RSBIT_THRESH 32
118
119/* Bit shift and mask */
120#define IXGBE_4_BIT_WIDTH  (CHAR_BIT / 2)
121#define IXGBE_4_BIT_MASK   RTE_LEN2MASK(IXGBE_4_BIT_WIDTH, uint8_t)
122#define IXGBE_8_BIT_WIDTH  CHAR_BIT
123#define IXGBE_8_BIT_MASK   UINT8_MAX
124
125#define IXGBEVF_PMD_NAME "rte_ixgbevf_pmd" /* PMD name */
126
127#define IXGBE_QUEUE_STAT_COUNTERS (sizeof(hw_stats->qprc) / sizeof(hw_stats->qprc[0]))
128
129#define IXGBE_HKEY_MAX_INDEX 10
130
131/* Additional timesync values. */
132#define NSEC_PER_SEC             1000000000L
133#define IXGBE_INCVAL_10GB        0x66666666
134#define IXGBE_INCVAL_1GB         0x40000000
135#define IXGBE_INCVAL_100         0x50000000
136#define IXGBE_INCVAL_SHIFT_10GB  28
137#define IXGBE_INCVAL_SHIFT_1GB   24
138#define IXGBE_INCVAL_SHIFT_100   21
139#define IXGBE_INCVAL_SHIFT_82599 7
140#define IXGBE_INCPER_SHIFT_82599 24
141
142#define IXGBE_CYCLECOUNTER_MASK   0xffffffffffffffffULL
143
144#define IXGBE_VT_CTL_POOLING_MODE_MASK         0x00030000
145#define IXGBE_VT_CTL_POOLING_MODE_ETAG         0x00010000
146#define DEFAULT_ETAG_ETYPE                     0x893f
147#define IXGBE_ETAG_ETYPE                       0x00005084
148#define IXGBE_ETAG_ETYPE_MASK                  0x0000ffff
149#define IXGBE_ETAG_ETYPE_VALID                 0x80000000
150#define IXGBE_RAH_ADTYPE                       0x40000000
151#define IXGBE_RAL_ETAG_FILTER_MASK             0x00003fff
152#define IXGBE_VMVIR_TAGA_MASK                  0x18000000
153#define IXGBE_VMVIR_TAGA_ETAG_INSERT           0x08000000
154#define IXGBE_VMTIR(_i) (0x00017000 + ((_i) * 4)) /* 64 of these (0-63) */
155#define IXGBE_QDE_STRIP_TAG                    0x00000004
156#define IXGBE_VTEICR_MASK                      0x07
157
158enum ixgbevf_xcast_modes {
159	IXGBEVF_XCAST_MODE_NONE = 0,
160	IXGBEVF_XCAST_MODE_MULTI,
161	IXGBEVF_XCAST_MODE_ALLMULTI,
162};
163
164#define IXGBE_EXVET_VET_EXT_SHIFT              16
165#define IXGBE_DMATXCTL_VT_MASK                 0xFFFF0000
166
167static int eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev);
168static int eth_ixgbe_dev_uninit(struct rte_eth_dev *eth_dev);
169static int ixgbe_fdir_filter_init(struct rte_eth_dev *eth_dev);
170static int ixgbe_fdir_filter_uninit(struct rte_eth_dev *eth_dev);
171static int ixgbe_l2_tn_filter_init(struct rte_eth_dev *eth_dev);
172static int ixgbe_l2_tn_filter_uninit(struct rte_eth_dev *eth_dev);
173static int ixgbe_ntuple_filter_uninit(struct rte_eth_dev *eth_dev);
174static int  ixgbe_dev_configure(struct rte_eth_dev *dev);
175static int  ixgbe_dev_start(struct rte_eth_dev *dev);
176static void ixgbe_dev_stop(struct rte_eth_dev *dev);
177static int  ixgbe_dev_set_link_up(struct rte_eth_dev *dev);
178static int  ixgbe_dev_set_link_down(struct rte_eth_dev *dev);
179static void ixgbe_dev_close(struct rte_eth_dev *dev);
180static void ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev);
181static void ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev);
182static void ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev);
183static void ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev);
184static int ixgbe_dev_link_update(struct rte_eth_dev *dev,
185				int wait_to_complete);
186static void ixgbe_dev_stats_get(struct rte_eth_dev *dev,
187				struct rte_eth_stats *stats);
188static int ixgbe_dev_xstats_get(struct rte_eth_dev *dev,
189				struct rte_eth_xstat *xstats, unsigned n);
190static int ixgbevf_dev_xstats_get(struct rte_eth_dev *dev,
191				  struct rte_eth_xstat *xstats, unsigned n);
192static void ixgbe_dev_stats_reset(struct rte_eth_dev *dev);
193static void ixgbe_dev_xstats_reset(struct rte_eth_dev *dev);
194static int ixgbe_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
195	struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit);
196static int ixgbevf_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
197	struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit);
198static int ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
199					     uint16_t queue_id,
200					     uint8_t stat_idx,
201					     uint8_t is_rx);
202static int ixgbe_fw_version_get(struct rte_eth_dev *dev, char *fw_version,
203				 size_t fw_size);
204static void ixgbe_dev_info_get(struct rte_eth_dev *dev,
205			       struct rte_eth_dev_info *dev_info);
206static const uint32_t *ixgbe_dev_supported_ptypes_get(struct rte_eth_dev *dev);
207static void ixgbevf_dev_info_get(struct rte_eth_dev *dev,
208				 struct rte_eth_dev_info *dev_info);
209static int ixgbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
210
211static int ixgbe_vlan_filter_set(struct rte_eth_dev *dev,
212		uint16_t vlan_id, int on);
213static int ixgbe_vlan_tpid_set(struct rte_eth_dev *dev,
214			       enum rte_vlan_type vlan_type,
215			       uint16_t tpid_id);
216static void ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev,
217		uint16_t queue, bool on);
218static void ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue,
219		int on);
220static void ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask);
221static void ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue);
222static void ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue);
223static void ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev);
224static void ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev);
225
226static int ixgbe_dev_led_on(struct rte_eth_dev *dev);
227static int ixgbe_dev_led_off(struct rte_eth_dev *dev);
228static int ixgbe_flow_ctrl_get(struct rte_eth_dev *dev,
229			       struct rte_eth_fc_conf *fc_conf);
230static int ixgbe_flow_ctrl_set(struct rte_eth_dev *dev,
231			       struct rte_eth_fc_conf *fc_conf);
232static int ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev,
233		struct rte_eth_pfc_conf *pfc_conf);
234static int ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
235			struct rte_eth_rss_reta_entry64 *reta_conf,
236			uint16_t reta_size);
237static int ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
238			struct rte_eth_rss_reta_entry64 *reta_conf,
239			uint16_t reta_size);
240static void ixgbe_dev_link_status_print(struct rte_eth_dev *dev);
241static int ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev);
242static int ixgbe_dev_macsec_interrupt_setup(struct rte_eth_dev *dev);
243static int ixgbe_dev_rxq_interrupt_setup(struct rte_eth_dev *dev);
244static int ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev);
245static int ixgbe_dev_interrupt_action(struct rte_eth_dev *dev,
246				      struct rte_intr_handle *handle);
247static void ixgbe_dev_interrupt_handler(struct rte_intr_handle *handle,
248		void *param);
249static void ixgbe_dev_interrupt_delayed_handler(void *param);
250static void ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
251		uint32_t index, uint32_t pool);
252static void ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index);
253static void ixgbe_set_default_mac_addr(struct rte_eth_dev *dev,
254					   struct ether_addr *mac_addr);
255static void ixgbe_dcb_init(struct ixgbe_hw *hw, struct ixgbe_dcb_config *dcb_config);
256static bool is_device_supported(struct rte_eth_dev *dev,
257				struct eth_driver *drv);
258
259/* For Virtual Function support */
260static int eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev);
261static int eth_ixgbevf_dev_uninit(struct rte_eth_dev *eth_dev);
262static int  ixgbevf_dev_configure(struct rte_eth_dev *dev);
263static int  ixgbevf_dev_start(struct rte_eth_dev *dev);
264static void ixgbevf_dev_stop(struct rte_eth_dev *dev);
265static void ixgbevf_dev_close(struct rte_eth_dev *dev);
266static void ixgbevf_intr_disable(struct ixgbe_hw *hw);
267static void ixgbevf_intr_enable(struct ixgbe_hw *hw);
268static void ixgbevf_dev_stats_get(struct rte_eth_dev *dev,
269		struct rte_eth_stats *stats);
270static void ixgbevf_dev_stats_reset(struct rte_eth_dev *dev);
271static int ixgbevf_vlan_filter_set(struct rte_eth_dev *dev,
272		uint16_t vlan_id, int on);
273static void ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev,
274		uint16_t queue, int on);
275static void ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask);
276static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on);
277static int ixgbevf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev,
278					    uint16_t queue_id);
279static int ixgbevf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev,
280					     uint16_t queue_id);
281static void ixgbevf_set_ivar_map(struct ixgbe_hw *hw, int8_t direction,
282				 uint8_t queue, uint8_t msix_vector);
283static void ixgbevf_configure_msix(struct rte_eth_dev *dev);
284static void ixgbevf_dev_allmulticast_enable(struct rte_eth_dev *dev);
285static void ixgbevf_dev_allmulticast_disable(struct rte_eth_dev *dev);
286
287/* For Eth VMDQ APIs support */
288static int ixgbe_uc_hash_table_set(struct rte_eth_dev *dev, struct
289		ether_addr * mac_addr, uint8_t on);
290static int ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev, uint8_t on);
291static int ixgbe_mirror_rule_set(struct rte_eth_dev *dev,
292		struct rte_eth_mirror_conf *mirror_conf,
293		uint8_t rule_id, uint8_t on);
294static int ixgbe_mirror_rule_reset(struct rte_eth_dev *dev,
295		uint8_t	rule_id);
296static int ixgbe_dev_rx_queue_intr_enable(struct rte_eth_dev *dev,
297					  uint16_t queue_id);
298static int ixgbe_dev_rx_queue_intr_disable(struct rte_eth_dev *dev,
299					   uint16_t queue_id);
300static void ixgbe_set_ivar_map(struct ixgbe_hw *hw, int8_t direction,
301			       uint8_t queue, uint8_t msix_vector);
302static void ixgbe_configure_msix(struct rte_eth_dev *dev);
303
304static int ixgbe_set_queue_rate_limit(struct rte_eth_dev *dev,
305		uint16_t queue_idx, uint16_t tx_rate);
306
307static void ixgbevf_add_mac_addr(struct rte_eth_dev *dev,
308				 struct ether_addr *mac_addr,
309				 uint32_t index, uint32_t pool);
310static void ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index);
311static void ixgbevf_set_default_mac_addr(struct rte_eth_dev *dev,
312					     struct ether_addr *mac_addr);
313static int ixgbe_syn_filter_get(struct rte_eth_dev *dev,
314			struct rte_eth_syn_filter *filter);
315static int ixgbe_syn_filter_handle(struct rte_eth_dev *dev,
316			enum rte_filter_op filter_op,
317			void *arg);
318static int ixgbe_add_5tuple_filter(struct rte_eth_dev *dev,
319			struct ixgbe_5tuple_filter *filter);
320static void ixgbe_remove_5tuple_filter(struct rte_eth_dev *dev,
321			struct ixgbe_5tuple_filter *filter);
322static int ixgbe_ntuple_filter_handle(struct rte_eth_dev *dev,
323				enum rte_filter_op filter_op,
324				void *arg);
325static int ixgbe_get_ntuple_filter(struct rte_eth_dev *dev,
326			struct rte_eth_ntuple_filter *filter);
327static int ixgbe_ethertype_filter_handle(struct rte_eth_dev *dev,
328				enum rte_filter_op filter_op,
329				void *arg);
330static int ixgbe_get_ethertype_filter(struct rte_eth_dev *dev,
331			struct rte_eth_ethertype_filter *filter);
332static int ixgbe_dev_filter_ctrl(struct rte_eth_dev *dev,
333		     enum rte_filter_type filter_type,
334		     enum rte_filter_op filter_op,
335		     void *arg);
336static int ixgbevf_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu);
337
338static int ixgbe_dev_set_mc_addr_list(struct rte_eth_dev *dev,
339				      struct ether_addr *mc_addr_set,
340				      uint32_t nb_mc_addr);
341static int ixgbe_dev_get_dcb_info(struct rte_eth_dev *dev,
342				   struct rte_eth_dcb_info *dcb_info);
343
344static int ixgbe_get_reg_length(struct rte_eth_dev *dev);
345static int ixgbe_get_regs(struct rte_eth_dev *dev,
346			    struct rte_dev_reg_info *regs);
347static int ixgbe_get_eeprom_length(struct rte_eth_dev *dev);
348static int ixgbe_get_eeprom(struct rte_eth_dev *dev,
349				struct rte_dev_eeprom_info *eeprom);
350static int ixgbe_set_eeprom(struct rte_eth_dev *dev,
351				struct rte_dev_eeprom_info *eeprom);
352
353static int ixgbevf_get_reg_length(struct rte_eth_dev *dev);
354static int ixgbevf_get_regs(struct rte_eth_dev *dev,
355				struct rte_dev_reg_info *regs);
356
357static int ixgbe_timesync_enable(struct rte_eth_dev *dev);
358static int ixgbe_timesync_disable(struct rte_eth_dev *dev);
359static int ixgbe_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
360					    struct timespec *timestamp,
361					    uint32_t flags);
362static int ixgbe_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
363					    struct timespec *timestamp);
364static int ixgbe_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta);
365static int ixgbe_timesync_read_time(struct rte_eth_dev *dev,
366				   struct timespec *timestamp);
367static int ixgbe_timesync_write_time(struct rte_eth_dev *dev,
368				   const struct timespec *timestamp);
369static void ixgbevf_dev_interrupt_handler(struct rte_intr_handle *handle,
370					  void *param);
371
372static int ixgbe_dev_l2_tunnel_eth_type_conf
373	(struct rte_eth_dev *dev, struct rte_eth_l2_tunnel_conf *l2_tunnel);
374static int ixgbe_dev_l2_tunnel_offload_set
375	(struct rte_eth_dev *dev,
376	 struct rte_eth_l2_tunnel_conf *l2_tunnel,
377	 uint32_t mask,
378	 uint8_t en);
379static int ixgbe_dev_l2_tunnel_filter_handle(struct rte_eth_dev *dev,
380					     enum rte_filter_op filter_op,
381					     void *arg);
382
383static int ixgbe_dev_udp_tunnel_port_add(struct rte_eth_dev *dev,
384					 struct rte_eth_udp_tunnel *udp_tunnel);
385static int ixgbe_dev_udp_tunnel_port_del(struct rte_eth_dev *dev,
386					 struct rte_eth_udp_tunnel *udp_tunnel);
387static int ixgbe_filter_restore(struct rte_eth_dev *dev);
388static void ixgbe_l2_tunnel_conf(struct rte_eth_dev *dev);
389
390/*
391 * Define VF Stats MACRO for Non "cleared on read" register
392 */
393#define UPDATE_VF_STAT(reg, last, cur)                          \
394{                                                               \
395	uint32_t latest = IXGBE_READ_REG(hw, reg);              \
396	cur += (latest - last) & UINT_MAX;                      \
397	last = latest;                                          \
398}
399
400#define UPDATE_VF_STAT_36BIT(lsb, msb, last, cur)                \
401{                                                                \
402	u64 new_lsb = IXGBE_READ_REG(hw, lsb);                   \
403	u64 new_msb = IXGBE_READ_REG(hw, msb);                   \
404	u64 latest = ((new_msb << 32) | new_lsb);                \
405	cur += (0x1000000000LL + latest - last) & 0xFFFFFFFFFLL; \
406	last = latest;                                           \
407}
408
409#define IXGBE_SET_HWSTRIP(h, q) do {\
410		uint32_t idx = (q) / (sizeof((h)->bitmap[0]) * NBBY); \
411		uint32_t bit = (q) % (sizeof((h)->bitmap[0]) * NBBY); \
412		(h)->bitmap[idx] |= 1 << bit;\
413	} while (0)
414
415#define IXGBE_CLEAR_HWSTRIP(h, q) do {\
416		uint32_t idx = (q) / (sizeof((h)->bitmap[0]) * NBBY); \
417		uint32_t bit = (q) % (sizeof((h)->bitmap[0]) * NBBY); \
418		(h)->bitmap[idx] &= ~(1 << bit);\
419	} while (0)
420
421#define IXGBE_GET_HWSTRIP(h, q, r) do {\
422		uint32_t idx = (q) / (sizeof((h)->bitmap[0]) * NBBY); \
423		uint32_t bit = (q) % (sizeof((h)->bitmap[0]) * NBBY); \
424		(r) = (h)->bitmap[idx] >> bit & 1;\
425	} while (0)
426
427/*
428 * The set of PCI devices this driver supports
429 */
430static const struct rte_pci_id pci_id_ixgbe_map[] = {
431	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598) },
432	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_BX) },
433	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_DUAL_PORT) },
434	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AF_SINGLE_PORT) },
435	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT) },
436	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598AT2) },
437	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_SFP_LOM) },
438	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_CX4) },
439	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_CX4_DUAL_PORT) },
440	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_DA_DUAL_PORT) },
441	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM) },
442	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82598EB_XF_LR) },
443	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4) },
444	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KX4_MEZZ) },
445	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_KR) },
446	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_COMBO_BACKPLANE) },
447	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_SUBDEV_ID_82599_KX4_KR_MEZZ) },
448	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_CX4) },
449	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP) },
450	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_SUBDEV_ID_82599_SFP) },
451	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_SUBDEV_ID_82599_RNDC) },
452	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_SUBDEV_ID_82599_560FLR) },
453	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_SUBDEV_ID_82599_ECNA_DP) },
454	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BACKPLANE_FCOE) },
455	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_FCOE) },
456	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_EM) },
457	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF2) },
458	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_SFP_SF_QP) },
459	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_QSFP_SF_QP) },
460	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599EN_SFP) },
461	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_XAUI_LOM) },
462	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_T3_LOM) },
463	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_LS) },
464	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T) },
465	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540T1) },
466	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_SFP) },
467	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_10G_T) },
468	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_1G_T) },
469	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T) },
470	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550T1) },
471	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR) },
472	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_KR_L) },
473	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP_N) },
474	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII) },
475	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SGMII_L) },
476	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_10G_T) },
477	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_QSFP) },
478	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_QSFP_N) },
479	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_SFP) },
480	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T) },
481	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_1G_T_L) },
482	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KX4) },
483	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_KR) },
484#ifdef RTE_NIC_BYPASS
485	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_BYPASS) },
486#endif
487	{ .vendor_id = 0, /* sentinel */ },
488};
489
490/*
491 * The set of PCI devices this driver supports (for 82599 VF)
492 */
493static const struct rte_pci_id pci_id_ixgbevf_map[] = {
494	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF) },
495	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_82599_VF_HV) },
496	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF) },
497	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X540_VF_HV) },
498	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF_HV) },
499	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550_VF) },
500	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF) },
501	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_A_VF_HV) },
502	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF) },
503	{ RTE_PCI_DEVICE(IXGBE_INTEL_VENDOR_ID, IXGBE_DEV_ID_X550EM_X_VF_HV) },
504	{ .vendor_id = 0, /* sentinel */ },
505};
506
507static const struct rte_eth_desc_lim rx_desc_lim = {
508	.nb_max = IXGBE_MAX_RING_DESC,
509	.nb_min = IXGBE_MIN_RING_DESC,
510	.nb_align = IXGBE_RXD_ALIGN,
511};
512
513static const struct rte_eth_desc_lim tx_desc_lim = {
514	.nb_max = IXGBE_MAX_RING_DESC,
515	.nb_min = IXGBE_MIN_RING_DESC,
516	.nb_align = IXGBE_TXD_ALIGN,
517	.nb_seg_max = IXGBE_TX_MAX_SEG,
518	.nb_mtu_seg_max = IXGBE_TX_MAX_SEG,
519};
520
521static const struct eth_dev_ops ixgbe_eth_dev_ops = {
522	.dev_configure        = ixgbe_dev_configure,
523	.dev_start            = ixgbe_dev_start,
524	.dev_stop             = ixgbe_dev_stop,
525	.dev_set_link_up    = ixgbe_dev_set_link_up,
526	.dev_set_link_down  = ixgbe_dev_set_link_down,
527	.dev_close            = ixgbe_dev_close,
528	.promiscuous_enable   = ixgbe_dev_promiscuous_enable,
529	.promiscuous_disable  = ixgbe_dev_promiscuous_disable,
530	.allmulticast_enable  = ixgbe_dev_allmulticast_enable,
531	.allmulticast_disable = ixgbe_dev_allmulticast_disable,
532	.link_update          = ixgbe_dev_link_update,
533	.stats_get            = ixgbe_dev_stats_get,
534	.xstats_get           = ixgbe_dev_xstats_get,
535	.stats_reset          = ixgbe_dev_stats_reset,
536	.xstats_reset         = ixgbe_dev_xstats_reset,
537	.xstats_get_names     = ixgbe_dev_xstats_get_names,
538	.queue_stats_mapping_set = ixgbe_dev_queue_stats_mapping_set,
539	.fw_version_get       = ixgbe_fw_version_get,
540	.dev_infos_get        = ixgbe_dev_info_get,
541	.dev_supported_ptypes_get = ixgbe_dev_supported_ptypes_get,
542	.mtu_set              = ixgbe_dev_mtu_set,
543	.vlan_filter_set      = ixgbe_vlan_filter_set,
544	.vlan_tpid_set        = ixgbe_vlan_tpid_set,
545	.vlan_offload_set     = ixgbe_vlan_offload_set,
546	.vlan_strip_queue_set = ixgbe_vlan_strip_queue_set,
547	.rx_queue_start	      = ixgbe_dev_rx_queue_start,
548	.rx_queue_stop        = ixgbe_dev_rx_queue_stop,
549	.tx_queue_start	      = ixgbe_dev_tx_queue_start,
550	.tx_queue_stop        = ixgbe_dev_tx_queue_stop,
551	.rx_queue_setup       = ixgbe_dev_rx_queue_setup,
552	.rx_queue_intr_enable = ixgbe_dev_rx_queue_intr_enable,
553	.rx_queue_intr_disable = ixgbe_dev_rx_queue_intr_disable,
554	.rx_queue_release     = ixgbe_dev_rx_queue_release,
555	.rx_queue_count       = ixgbe_dev_rx_queue_count,
556	.rx_descriptor_done   = ixgbe_dev_rx_descriptor_done,
557	.tx_queue_setup       = ixgbe_dev_tx_queue_setup,
558	.tx_queue_release     = ixgbe_dev_tx_queue_release,
559	.dev_led_on           = ixgbe_dev_led_on,
560	.dev_led_off          = ixgbe_dev_led_off,
561	.flow_ctrl_get        = ixgbe_flow_ctrl_get,
562	.flow_ctrl_set        = ixgbe_flow_ctrl_set,
563	.priority_flow_ctrl_set = ixgbe_priority_flow_ctrl_set,
564	.mac_addr_add         = ixgbe_add_rar,
565	.mac_addr_remove      = ixgbe_remove_rar,
566	.mac_addr_set         = ixgbe_set_default_mac_addr,
567	.uc_hash_table_set    = ixgbe_uc_hash_table_set,
568	.uc_all_hash_table_set  = ixgbe_uc_all_hash_table_set,
569	.mirror_rule_set      = ixgbe_mirror_rule_set,
570	.mirror_rule_reset    = ixgbe_mirror_rule_reset,
571	.set_queue_rate_limit = ixgbe_set_queue_rate_limit,
572	.reta_update          = ixgbe_dev_rss_reta_update,
573	.reta_query           = ixgbe_dev_rss_reta_query,
574#ifdef RTE_NIC_BYPASS
575	.bypass_init          = ixgbe_bypass_init,
576	.bypass_state_set     = ixgbe_bypass_state_store,
577	.bypass_state_show    = ixgbe_bypass_state_show,
578	.bypass_event_set     = ixgbe_bypass_event_store,
579	.bypass_event_show    = ixgbe_bypass_event_show,
580	.bypass_wd_timeout_set  = ixgbe_bypass_wd_timeout_store,
581	.bypass_wd_timeout_show = ixgbe_bypass_wd_timeout_show,
582	.bypass_ver_show      = ixgbe_bypass_ver_show,
583	.bypass_wd_reset      = ixgbe_bypass_wd_reset,
584#endif /* RTE_NIC_BYPASS */
585	.rss_hash_update      = ixgbe_dev_rss_hash_update,
586	.rss_hash_conf_get    = ixgbe_dev_rss_hash_conf_get,
587	.filter_ctrl          = ixgbe_dev_filter_ctrl,
588	.set_mc_addr_list     = ixgbe_dev_set_mc_addr_list,
589	.rxq_info_get         = ixgbe_rxq_info_get,
590	.txq_info_get         = ixgbe_txq_info_get,
591	.timesync_enable      = ixgbe_timesync_enable,
592	.timesync_disable     = ixgbe_timesync_disable,
593	.timesync_read_rx_timestamp = ixgbe_timesync_read_rx_timestamp,
594	.timesync_read_tx_timestamp = ixgbe_timesync_read_tx_timestamp,
595	.get_reg              = ixgbe_get_regs,
596	.get_eeprom_length    = ixgbe_get_eeprom_length,
597	.get_eeprom           = ixgbe_get_eeprom,
598	.set_eeprom           = ixgbe_set_eeprom,
599	.get_dcb_info         = ixgbe_dev_get_dcb_info,
600	.timesync_adjust_time = ixgbe_timesync_adjust_time,
601	.timesync_read_time   = ixgbe_timesync_read_time,
602	.timesync_write_time  = ixgbe_timesync_write_time,
603	.l2_tunnel_eth_type_conf = ixgbe_dev_l2_tunnel_eth_type_conf,
604	.l2_tunnel_offload_set   = ixgbe_dev_l2_tunnel_offload_set,
605	.udp_tunnel_port_add  = ixgbe_dev_udp_tunnel_port_add,
606	.udp_tunnel_port_del  = ixgbe_dev_udp_tunnel_port_del,
607};
608
609/*
610 * dev_ops for virtual function, bare necessities for basic vf
611 * operation have been implemented
612 */
613static const struct eth_dev_ops ixgbevf_eth_dev_ops = {
614	.dev_configure        = ixgbevf_dev_configure,
615	.dev_start            = ixgbevf_dev_start,
616	.dev_stop             = ixgbevf_dev_stop,
617	.link_update          = ixgbe_dev_link_update,
618	.stats_get            = ixgbevf_dev_stats_get,
619	.xstats_get           = ixgbevf_dev_xstats_get,
620	.stats_reset          = ixgbevf_dev_stats_reset,
621	.xstats_reset         = ixgbevf_dev_stats_reset,
622	.xstats_get_names     = ixgbevf_dev_xstats_get_names,
623	.dev_close            = ixgbevf_dev_close,
624	.allmulticast_enable  = ixgbevf_dev_allmulticast_enable,
625	.allmulticast_disable = ixgbevf_dev_allmulticast_disable,
626	.dev_infos_get        = ixgbevf_dev_info_get,
627	.dev_supported_ptypes_get = ixgbe_dev_supported_ptypes_get,
628	.mtu_set              = ixgbevf_dev_set_mtu,
629	.vlan_filter_set      = ixgbevf_vlan_filter_set,
630	.vlan_strip_queue_set = ixgbevf_vlan_strip_queue_set,
631	.vlan_offload_set     = ixgbevf_vlan_offload_set,
632	.rx_queue_setup       = ixgbe_dev_rx_queue_setup,
633	.rx_queue_release     = ixgbe_dev_rx_queue_release,
634	.rx_descriptor_done   = ixgbe_dev_rx_descriptor_done,
635	.tx_queue_setup       = ixgbe_dev_tx_queue_setup,
636	.tx_queue_release     = ixgbe_dev_tx_queue_release,
637	.rx_queue_intr_enable = ixgbevf_dev_rx_queue_intr_enable,
638	.rx_queue_intr_disable = ixgbevf_dev_rx_queue_intr_disable,
639	.mac_addr_add         = ixgbevf_add_mac_addr,
640	.mac_addr_remove      = ixgbevf_remove_mac_addr,
641	.set_mc_addr_list     = ixgbe_dev_set_mc_addr_list,
642	.rxq_info_get         = ixgbe_rxq_info_get,
643	.txq_info_get         = ixgbe_txq_info_get,
644	.mac_addr_set         = ixgbevf_set_default_mac_addr,
645	.get_reg              = ixgbevf_get_regs,
646	.reta_update          = ixgbe_dev_rss_reta_update,
647	.reta_query           = ixgbe_dev_rss_reta_query,
648	.rss_hash_update      = ixgbe_dev_rss_hash_update,
649	.rss_hash_conf_get    = ixgbe_dev_rss_hash_conf_get,
650};
651
652/* store statistics names and its offset in stats structure */
653struct rte_ixgbe_xstats_name_off {
654	char name[RTE_ETH_XSTATS_NAME_SIZE];
655	unsigned offset;
656};
657
658static const struct rte_ixgbe_xstats_name_off rte_ixgbe_stats_strings[] = {
659	{"rx_crc_errors", offsetof(struct ixgbe_hw_stats, crcerrs)},
660	{"rx_illegal_byte_errors", offsetof(struct ixgbe_hw_stats, illerrc)},
661	{"rx_error_bytes", offsetof(struct ixgbe_hw_stats, errbc)},
662	{"mac_local_errors", offsetof(struct ixgbe_hw_stats, mlfc)},
663	{"mac_remote_errors", offsetof(struct ixgbe_hw_stats, mrfc)},
664	{"rx_length_errors", offsetof(struct ixgbe_hw_stats, rlec)},
665	{"tx_xon_packets", offsetof(struct ixgbe_hw_stats, lxontxc)},
666	{"rx_xon_packets", offsetof(struct ixgbe_hw_stats, lxonrxc)},
667	{"tx_xoff_packets", offsetof(struct ixgbe_hw_stats, lxofftxc)},
668	{"rx_xoff_packets", offsetof(struct ixgbe_hw_stats, lxoffrxc)},
669	{"rx_size_64_packets", offsetof(struct ixgbe_hw_stats, prc64)},
670	{"rx_size_65_to_127_packets", offsetof(struct ixgbe_hw_stats, prc127)},
671	{"rx_size_128_to_255_packets", offsetof(struct ixgbe_hw_stats, prc255)},
672	{"rx_size_256_to_511_packets", offsetof(struct ixgbe_hw_stats, prc511)},
673	{"rx_size_512_to_1023_packets", offsetof(struct ixgbe_hw_stats,
674		prc1023)},
675	{"rx_size_1024_to_max_packets", offsetof(struct ixgbe_hw_stats,
676		prc1522)},
677	{"rx_broadcast_packets", offsetof(struct ixgbe_hw_stats, bprc)},
678	{"rx_multicast_packets", offsetof(struct ixgbe_hw_stats, mprc)},
679	{"rx_fragment_errors", offsetof(struct ixgbe_hw_stats, rfc)},
680	{"rx_undersize_errors", offsetof(struct ixgbe_hw_stats, ruc)},
681	{"rx_oversize_errors", offsetof(struct ixgbe_hw_stats, roc)},
682	{"rx_jabber_errors", offsetof(struct ixgbe_hw_stats, rjc)},
683	{"rx_management_packets", offsetof(struct ixgbe_hw_stats, mngprc)},
684	{"rx_management_dropped", offsetof(struct ixgbe_hw_stats, mngpdc)},
685	{"tx_management_packets", offsetof(struct ixgbe_hw_stats, mngptc)},
686	{"rx_total_packets", offsetof(struct ixgbe_hw_stats, tpr)},
687	{"rx_total_bytes", offsetof(struct ixgbe_hw_stats, tor)},
688	{"tx_total_packets", offsetof(struct ixgbe_hw_stats, tpt)},
689	{"tx_size_64_packets", offsetof(struct ixgbe_hw_stats, ptc64)},
690	{"tx_size_65_to_127_packets", offsetof(struct ixgbe_hw_stats, ptc127)},
691	{"tx_size_128_to_255_packets", offsetof(struct ixgbe_hw_stats, ptc255)},
692	{"tx_size_256_to_511_packets", offsetof(struct ixgbe_hw_stats, ptc511)},
693	{"tx_size_512_to_1023_packets", offsetof(struct ixgbe_hw_stats,
694		ptc1023)},
695	{"tx_size_1024_to_max_packets", offsetof(struct ixgbe_hw_stats,
696		ptc1522)},
697	{"tx_multicast_packets", offsetof(struct ixgbe_hw_stats, mptc)},
698	{"tx_broadcast_packets", offsetof(struct ixgbe_hw_stats, bptc)},
699	{"rx_mac_short_packet_dropped", offsetof(struct ixgbe_hw_stats, mspdc)},
700	{"rx_l3_l4_xsum_error", offsetof(struct ixgbe_hw_stats, xec)},
701
702	{"flow_director_added_filters", offsetof(struct ixgbe_hw_stats,
703		fdirustat_add)},
704	{"flow_director_removed_filters", offsetof(struct ixgbe_hw_stats,
705		fdirustat_remove)},
706	{"flow_director_filter_add_errors", offsetof(struct ixgbe_hw_stats,
707		fdirfstat_fadd)},
708	{"flow_director_filter_remove_errors", offsetof(struct ixgbe_hw_stats,
709		fdirfstat_fremove)},
710	{"flow_director_matched_filters", offsetof(struct ixgbe_hw_stats,
711		fdirmatch)},
712	{"flow_director_missed_filters", offsetof(struct ixgbe_hw_stats,
713		fdirmiss)},
714
715	{"rx_fcoe_crc_errors", offsetof(struct ixgbe_hw_stats, fccrc)},
716	{"rx_fcoe_dropped", offsetof(struct ixgbe_hw_stats, fcoerpdc)},
717	{"rx_fcoe_mbuf_allocation_errors", offsetof(struct ixgbe_hw_stats,
718		fclast)},
719	{"rx_fcoe_packets", offsetof(struct ixgbe_hw_stats, fcoeprc)},
720	{"tx_fcoe_packets", offsetof(struct ixgbe_hw_stats, fcoeptc)},
721	{"rx_fcoe_bytes", offsetof(struct ixgbe_hw_stats, fcoedwrc)},
722	{"tx_fcoe_bytes", offsetof(struct ixgbe_hw_stats, fcoedwtc)},
723	{"rx_fcoe_no_direct_data_placement", offsetof(struct ixgbe_hw_stats,
724		fcoe_noddp)},
725	{"rx_fcoe_no_direct_data_placement_ext_buff",
726		offsetof(struct ixgbe_hw_stats, fcoe_noddp_ext_buff)},
727
728	{"tx_flow_control_xon_packets", offsetof(struct ixgbe_hw_stats,
729		lxontxc)},
730	{"rx_flow_control_xon_packets", offsetof(struct ixgbe_hw_stats,
731		lxonrxc)},
732	{"tx_flow_control_xoff_packets", offsetof(struct ixgbe_hw_stats,
733		lxofftxc)},
734	{"rx_flow_control_xoff_packets", offsetof(struct ixgbe_hw_stats,
735		lxoffrxc)},
736	{"rx_total_missed_packets", offsetof(struct ixgbe_hw_stats, mpctotal)},
737};
738
739#define IXGBE_NB_HW_STATS (sizeof(rte_ixgbe_stats_strings) / \
740			   sizeof(rte_ixgbe_stats_strings[0]))
741
742/* MACsec statistics */
743static const struct rte_ixgbe_xstats_name_off rte_ixgbe_macsec_strings[] = {
744	{"out_pkts_untagged", offsetof(struct ixgbe_macsec_stats,
745		out_pkts_untagged)},
746	{"out_pkts_encrypted", offsetof(struct ixgbe_macsec_stats,
747		out_pkts_encrypted)},
748	{"out_pkts_protected", offsetof(struct ixgbe_macsec_stats,
749		out_pkts_protected)},
750	{"out_octets_encrypted", offsetof(struct ixgbe_macsec_stats,
751		out_octets_encrypted)},
752	{"out_octets_protected", offsetof(struct ixgbe_macsec_stats,
753		out_octets_protected)},
754	{"in_pkts_untagged", offsetof(struct ixgbe_macsec_stats,
755		in_pkts_untagged)},
756	{"in_pkts_badtag", offsetof(struct ixgbe_macsec_stats,
757		in_pkts_badtag)},
758	{"in_pkts_nosci", offsetof(struct ixgbe_macsec_stats,
759		in_pkts_nosci)},
760	{"in_pkts_unknownsci", offsetof(struct ixgbe_macsec_stats,
761		in_pkts_unknownsci)},
762	{"in_octets_decrypted", offsetof(struct ixgbe_macsec_stats,
763		in_octets_decrypted)},
764	{"in_octets_validated", offsetof(struct ixgbe_macsec_stats,
765		in_octets_validated)},
766	{"in_pkts_unchecked", offsetof(struct ixgbe_macsec_stats,
767		in_pkts_unchecked)},
768	{"in_pkts_delayed", offsetof(struct ixgbe_macsec_stats,
769		in_pkts_delayed)},
770	{"in_pkts_late", offsetof(struct ixgbe_macsec_stats,
771		in_pkts_late)},
772	{"in_pkts_ok", offsetof(struct ixgbe_macsec_stats,
773		in_pkts_ok)},
774	{"in_pkts_invalid", offsetof(struct ixgbe_macsec_stats,
775		in_pkts_invalid)},
776	{"in_pkts_notvalid", offsetof(struct ixgbe_macsec_stats,
777		in_pkts_notvalid)},
778	{"in_pkts_unusedsa", offsetof(struct ixgbe_macsec_stats,
779		in_pkts_unusedsa)},
780	{"in_pkts_notusingsa", offsetof(struct ixgbe_macsec_stats,
781		in_pkts_notusingsa)},
782};
783
784#define IXGBE_NB_MACSEC_STATS (sizeof(rte_ixgbe_macsec_strings) / \
785			   sizeof(rte_ixgbe_macsec_strings[0]))
786
787/* Per-queue statistics */
788static const struct rte_ixgbe_xstats_name_off rte_ixgbe_rxq_strings[] = {
789	{"mbuf_allocation_errors", offsetof(struct ixgbe_hw_stats, rnbc)},
790	{"dropped", offsetof(struct ixgbe_hw_stats, mpc)},
791	{"xon_packets", offsetof(struct ixgbe_hw_stats, pxonrxc)},
792	{"xoff_packets", offsetof(struct ixgbe_hw_stats, pxoffrxc)},
793};
794
795#define IXGBE_NB_RXQ_PRIO_STATS (sizeof(rte_ixgbe_rxq_strings) / \
796			   sizeof(rte_ixgbe_rxq_strings[0]))
797#define IXGBE_NB_RXQ_PRIO_VALUES 8
798
799static const struct rte_ixgbe_xstats_name_off rte_ixgbe_txq_strings[] = {
800	{"xon_packets", offsetof(struct ixgbe_hw_stats, pxontxc)},
801	{"xoff_packets", offsetof(struct ixgbe_hw_stats, pxofftxc)},
802	{"xon_to_xoff_packets", offsetof(struct ixgbe_hw_stats,
803		pxon2offc)},
804};
805
806#define IXGBE_NB_TXQ_PRIO_STATS (sizeof(rte_ixgbe_txq_strings) / \
807			   sizeof(rte_ixgbe_txq_strings[0]))
808#define IXGBE_NB_TXQ_PRIO_VALUES 8
809
810static const struct rte_ixgbe_xstats_name_off rte_ixgbevf_stats_strings[] = {
811	{"rx_multicast_packets", offsetof(struct ixgbevf_hw_stats, vfmprc)},
812};
813
814#define IXGBEVF_NB_XSTATS (sizeof(rte_ixgbevf_stats_strings) /	\
815		sizeof(rte_ixgbevf_stats_strings[0]))
816
817/**
818 * Atomically reads the link status information from global
819 * structure rte_eth_dev.
820 *
821 * @param dev
822 *   - Pointer to the structure rte_eth_dev to read from.
823 *   - Pointer to the buffer to be saved with the link status.
824 *
825 * @return
826 *   - On success, zero.
827 *   - On failure, negative value.
828 */
829static inline int
830rte_ixgbe_dev_atomic_read_link_status(struct rte_eth_dev *dev,
831				struct rte_eth_link *link)
832{
833	struct rte_eth_link *dst = link;
834	struct rte_eth_link *src = &(dev->data->dev_link);
835
836	if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
837					*(uint64_t *)src) == 0)
838		return -1;
839
840	return 0;
841}
842
843/**
844 * Atomically writes the link status information into global
845 * structure rte_eth_dev.
846 *
847 * @param dev
848 *   - Pointer to the structure rte_eth_dev to read from.
849 *   - Pointer to the buffer to be saved with the link status.
850 *
851 * @return
852 *   - On success, zero.
853 *   - On failure, negative value.
854 */
855static inline int
856rte_ixgbe_dev_atomic_write_link_status(struct rte_eth_dev *dev,
857				struct rte_eth_link *link)
858{
859	struct rte_eth_link *dst = &(dev->data->dev_link);
860	struct rte_eth_link *src = link;
861
862	if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
863					*(uint64_t *)src) == 0)
864		return -1;
865
866	return 0;
867}
868
869/*
870 * This function is the same as ixgbe_is_sfp() in base/ixgbe.h.
871 */
872static inline int
873ixgbe_is_sfp(struct ixgbe_hw *hw)
874{
875	switch (hw->phy.type) {
876	case ixgbe_phy_sfp_avago:
877	case ixgbe_phy_sfp_ftl:
878	case ixgbe_phy_sfp_intel:
879	case ixgbe_phy_sfp_unknown:
880	case ixgbe_phy_sfp_passive_tyco:
881	case ixgbe_phy_sfp_passive_unknown:
882		return 1;
883	default:
884		return 0;
885	}
886}
887
888static inline int32_t
889ixgbe_pf_reset_hw(struct ixgbe_hw *hw)
890{
891	uint32_t ctrl_ext;
892	int32_t status;
893
894	status = ixgbe_reset_hw(hw);
895
896	ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
897	/* Set PF Reset Done bit so PF/VF Mail Ops can work */
898	ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
899	IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
900	IXGBE_WRITE_FLUSH(hw);
901
902	if (status == IXGBE_ERR_SFP_NOT_PRESENT)
903		status = IXGBE_SUCCESS;
904	return status;
905}
906
907static inline void
908ixgbe_enable_intr(struct rte_eth_dev *dev)
909{
910	struct ixgbe_interrupt *intr =
911		IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
912	struct ixgbe_hw *hw =
913		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
914
915	IXGBE_WRITE_REG(hw, IXGBE_EIMS, intr->mask);
916	IXGBE_WRITE_FLUSH(hw);
917}
918
919/*
920 * This function is based on ixgbe_disable_intr() in base/ixgbe.h.
921 */
922static void
923ixgbe_disable_intr(struct ixgbe_hw *hw)
924{
925	PMD_INIT_FUNC_TRACE();
926
927	if (hw->mac.type == ixgbe_mac_82598EB) {
928		IXGBE_WRITE_REG(hw, IXGBE_EIMC, ~0);
929	} else {
930		IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xFFFF0000);
931		IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), ~0);
932		IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), ~0);
933	}
934	IXGBE_WRITE_FLUSH(hw);
935}
936
937/*
938 * This function resets queue statistics mapping registers.
939 * From Niantic datasheet, Initialization of Statistics section:
940 * "...if software requires the queue counters, the RQSMR and TQSM registers
941 * must be re-programmed following a device reset.
942 */
943static void
944ixgbe_reset_qstat_mappings(struct ixgbe_hw *hw)
945{
946	uint32_t i;
947
948	for (i = 0; i != IXGBE_NB_STAT_MAPPING_REGS; i++) {
949		IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), 0);
950		IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), 0);
951	}
952}
953
954
955static int
956ixgbe_dev_queue_stats_mapping_set(struct rte_eth_dev *eth_dev,
957				  uint16_t queue_id,
958				  uint8_t stat_idx,
959				  uint8_t is_rx)
960{
961#define QSM_REG_NB_BITS_PER_QMAP_FIELD 8
962#define NB_QMAP_FIELDS_PER_QSM_REG 4
963#define QMAP_FIELD_RESERVED_BITS_MASK 0x0f
964
965	struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
966	struct ixgbe_stat_mapping_registers *stat_mappings =
967		IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(eth_dev->data->dev_private);
968	uint32_t qsmr_mask = 0;
969	uint32_t clearing_mask = QMAP_FIELD_RESERVED_BITS_MASK;
970	uint32_t q_map;
971	uint8_t n, offset;
972
973	if ((hw->mac.type != ixgbe_mac_82599EB) &&
974		(hw->mac.type != ixgbe_mac_X540) &&
975		(hw->mac.type != ixgbe_mac_X550) &&
976		(hw->mac.type != ixgbe_mac_X550EM_x) &&
977		(hw->mac.type != ixgbe_mac_X550EM_a))
978		return -ENOSYS;
979
980	PMD_INIT_LOG(DEBUG, "Setting port %d, %s queue_id %d to stat index %d",
981		     (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX",
982		     queue_id, stat_idx);
983
984	n = (uint8_t)(queue_id / NB_QMAP_FIELDS_PER_QSM_REG);
985	if (n >= IXGBE_NB_STAT_MAPPING_REGS) {
986		PMD_INIT_LOG(ERR, "Nb of stat mapping registers exceeded");
987		return -EIO;
988	}
989	offset = (uint8_t)(queue_id % NB_QMAP_FIELDS_PER_QSM_REG);
990
991	/* Now clear any previous stat_idx set */
992	clearing_mask <<= (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
993	if (!is_rx)
994		stat_mappings->tqsm[n] &= ~clearing_mask;
995	else
996		stat_mappings->rqsmr[n] &= ~clearing_mask;
997
998	q_map = (uint32_t)stat_idx;
999	q_map &= QMAP_FIELD_RESERVED_BITS_MASK;
1000	qsmr_mask = q_map << (QSM_REG_NB_BITS_PER_QMAP_FIELD * offset);
1001	if (!is_rx)
1002		stat_mappings->tqsm[n] |= qsmr_mask;
1003	else
1004		stat_mappings->rqsmr[n] |= qsmr_mask;
1005
1006	PMD_INIT_LOG(DEBUG, "Set port %d, %s queue_id %d to stat index %d",
1007		     (int)(eth_dev->data->port_id), is_rx ? "RX" : "TX",
1008		     queue_id, stat_idx);
1009	PMD_INIT_LOG(DEBUG, "%s[%d] = 0x%08x", is_rx ? "RQSMR" : "TQSM", n,
1010		     is_rx ? stat_mappings->rqsmr[n] : stat_mappings->tqsm[n]);
1011
1012	/* Now write the mapping in the appropriate register */
1013	if (is_rx) {
1014		PMD_INIT_LOG(DEBUG, "Write 0x%x to RX IXGBE stat mapping reg:%d",
1015			     stat_mappings->rqsmr[n], n);
1016		IXGBE_WRITE_REG(hw, IXGBE_RQSMR(n), stat_mappings->rqsmr[n]);
1017	} else {
1018		PMD_INIT_LOG(DEBUG, "Write 0x%x to TX IXGBE stat mapping reg:%d",
1019			     stat_mappings->tqsm[n], n);
1020		IXGBE_WRITE_REG(hw, IXGBE_TQSM(n), stat_mappings->tqsm[n]);
1021	}
1022	return 0;
1023}
1024
1025static void
1026ixgbe_restore_statistics_mapping(struct rte_eth_dev *dev)
1027{
1028	struct ixgbe_stat_mapping_registers *stat_mappings =
1029		IXGBE_DEV_PRIVATE_TO_STAT_MAPPINGS(dev->data->dev_private);
1030	struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1031	int i;
1032
1033	/* write whatever was in stat mapping table to the NIC */
1034	for (i = 0; i < IXGBE_NB_STAT_MAPPING_REGS; i++) {
1035		/* rx */
1036		IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i), stat_mappings->rqsmr[i]);
1037
1038		/* tx */
1039		IXGBE_WRITE_REG(hw, IXGBE_TQSM(i), stat_mappings->tqsm[i]);
1040	}
1041}
1042
1043static void
1044ixgbe_dcb_init(struct ixgbe_hw *hw, struct ixgbe_dcb_config *dcb_config)
1045{
1046	uint8_t i;
1047	struct ixgbe_dcb_tc_config *tc;
1048	uint8_t dcb_max_tc = IXGBE_DCB_MAX_TRAFFIC_CLASS;
1049
1050	dcb_config->num_tcs.pg_tcs = dcb_max_tc;
1051	dcb_config->num_tcs.pfc_tcs = dcb_max_tc;
1052	for (i = 0; i < dcb_max_tc; i++) {
1053		tc = &dcb_config->tc_config[i];
1054		tc->path[IXGBE_DCB_TX_CONFIG].bwg_id = i;
1055		tc->path[IXGBE_DCB_TX_CONFIG].bwg_percent =
1056				 (uint8_t)(100/dcb_max_tc + (i & 1));
1057		tc->path[IXGBE_DCB_RX_CONFIG].bwg_id = i;
1058		tc->path[IXGBE_DCB_RX_CONFIG].bwg_percent =
1059				 (uint8_t)(100/dcb_max_tc + (i & 1));
1060		tc->pfc = ixgbe_dcb_pfc_disabled;
1061	}
1062
1063	/* Initialize default user to priority mapping, UPx->TC0 */
1064	tc = &dcb_config->tc_config[0];
1065	tc->path[IXGBE_DCB_TX_CONFIG].up_to_tc_bitmap = 0xFF;
1066	tc->path[IXGBE_DCB_RX_CONFIG].up_to_tc_bitmap = 0xFF;
1067	for (i = 0; i < IXGBE_DCB_MAX_BW_GROUP; i++) {
1068		dcb_config->bw_percentage[IXGBE_DCB_TX_CONFIG][i] = 100;
1069		dcb_config->bw_percentage[IXGBE_DCB_RX_CONFIG][i] = 100;
1070	}
1071	dcb_config->rx_pba_cfg = ixgbe_dcb_pba_equal;
1072	dcb_config->pfc_mode_enable = false;
1073	dcb_config->vt_mode = true;
1074	dcb_config->round_robin_enable = false;
1075	/* support all DCB capabilities in 82599 */
1076	dcb_config->support.capabilities = 0xFF;
1077
1078	/*we only support 4 Tcs for X540, X550 */
1079	if (hw->mac.type == ixgbe_mac_X540 ||
1080		hw->mac.type == ixgbe_mac_X550 ||
1081		hw->mac.type == ixgbe_mac_X550EM_x ||
1082		hw->mac.type == ixgbe_mac_X550EM_a) {
1083		dcb_config->num_tcs.pg_tcs = 4;
1084		dcb_config->num_tcs.pfc_tcs = 4;
1085	}
1086}
1087
1088/*
1089 * Ensure that all locks are released before first NVM or PHY access
1090 */
1091static void
1092ixgbe_swfw_lock_reset(struct ixgbe_hw *hw)
1093{
1094	uint16_t mask;
1095
1096	/*
1097	 * Phy lock should not fail in this early stage. If this is the case,
1098	 * it is due to an improper exit of the application.
1099	 * So force the release of the faulty lock. Release of common lock
1100	 * is done automatically by swfw_sync function.
1101	 */
1102	mask = IXGBE_GSSR_PHY0_SM << hw->bus.func;
1103	if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) {
1104		PMD_DRV_LOG(DEBUG, "SWFW phy%d lock released", hw->bus.func);
1105	}
1106	ixgbe_release_swfw_semaphore(hw, mask);
1107
1108	/*
1109	 * These ones are more tricky since they are common to all ports; but
1110	 * swfw_sync retries last long enough (1s) to be almost sure that if
1111	 * lock can not be taken it is due to an improper lock of the
1112	 * semaphore.
1113	 */
1114	mask = IXGBE_GSSR_EEP_SM | IXGBE_GSSR_MAC_CSR_SM | IXGBE_GSSR_SW_MNG_SM;
1115	if (ixgbe_acquire_swfw_semaphore(hw, mask) < 0) {
1116		PMD_DRV_LOG(DEBUG, "SWFW common locks released");
1117	}
1118	ixgbe_release_swfw_semaphore(hw, mask);
1119}
1120
1121/*
1122 * This function is based on code in ixgbe_attach() in base/ixgbe.c.
1123 * It returns 0 on success.
1124 */
1125static int
1126eth_ixgbe_dev_init(struct rte_eth_dev *eth_dev)
1127{
1128	struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(eth_dev);
1129	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
1130	struct ixgbe_hw *hw =
1131		IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1132	struct ixgbe_vfta *shadow_vfta =
1133		IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
1134	struct ixgbe_hwstrip *hwstrip =
1135		IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
1136	struct ixgbe_dcb_config *dcb_config =
1137		IXGBE_DEV_PRIVATE_TO_DCB_CFG(eth_dev->data->dev_private);
1138	struct ixgbe_filter_info *filter_info =
1139		IXGBE_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private);
1140	uint32_t ctrl_ext;
1141	uint16_t csum;
1142	int diag, i;
1143
1144	PMD_INIT_FUNC_TRACE();
1145
1146	eth_dev->dev_ops = &ixgbe_eth_dev_ops;
1147	eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
1148	eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
1149	eth_dev->tx_pkt_prepare = &ixgbe_prep_pkts;
1150
1151	/*
1152	 * For secondary processes, we don't initialise any further as primary
1153	 * has already done this work. Only check we don't need a different
1154	 * RX and TX function.
1155	 */
1156	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
1157		struct ixgbe_tx_queue *txq;
1158		/* TX queue function in primary, set by last queue initialized
1159		 * Tx queue may not initialized by primary process
1160		 */
1161		if (eth_dev->data->tx_queues) {
1162			txq = eth_dev->data->tx_queues[eth_dev->data->nb_tx_queues-1];
1163			ixgbe_set_tx_function(eth_dev, txq);
1164		} else {
1165			/* Use default TX function if we get here */
1166			PMD_INIT_LOG(NOTICE, "No TX queues configured yet. "
1167				     "Using default TX function.");
1168		}
1169
1170		ixgbe_set_rx_function(eth_dev);
1171
1172		return 0;
1173	}
1174
1175	rte_eth_copy_pci_info(eth_dev, pci_dev);
1176	eth_dev->data->dev_flags |= RTE_ETH_DEV_DETACHABLE;
1177
1178	/* Vendor and Device ID need to be set before init of shared code */
1179	hw->device_id = pci_dev->id.device_id;
1180	hw->vendor_id = pci_dev->id.vendor_id;
1181	hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
1182	hw->allow_unsupported_sfp = 1;
1183
1184	/* Initialize the shared code (base driver) */
1185#ifdef RTE_NIC_BYPASS
1186	diag = ixgbe_bypass_init_shared_code(hw);
1187#else
1188	diag = ixgbe_init_shared_code(hw);
1189#endif /* RTE_NIC_BYPASS */
1190
1191	if (diag != IXGBE_SUCCESS) {
1192		PMD_INIT_LOG(ERR, "Shared code init failed: %d", diag);
1193		return -EIO;
1194	}
1195
1196	/* pick up the PCI bus settings for reporting later */
1197	ixgbe_get_bus_info(hw);
1198
1199	/* Unlock any pending hardware semaphore */
1200	ixgbe_swfw_lock_reset(hw);
1201
1202	/* Initialize DCB configuration*/
1203	memset(dcb_config, 0, sizeof(struct ixgbe_dcb_config));
1204	ixgbe_dcb_init(hw, dcb_config);
1205	/* Get Hardware Flow Control setting */
1206	hw->fc.requested_mode = ixgbe_fc_full;
1207	hw->fc.current_mode = ixgbe_fc_full;
1208	hw->fc.pause_time = IXGBE_FC_PAUSE;
1209	for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
1210		hw->fc.low_water[i] = IXGBE_FC_LO;
1211		hw->fc.high_water[i] = IXGBE_FC_HI;
1212	}
1213	hw->fc.send_xon = 1;
1214
1215	/* Make sure we have a good EEPROM before we read from it */
1216	diag = ixgbe_validate_eeprom_checksum(hw, &csum);
1217	if (diag != IXGBE_SUCCESS) {
1218		PMD_INIT_LOG(ERR, "The EEPROM checksum is not valid: %d", diag);
1219		return -EIO;
1220	}
1221
1222#ifdef RTE_NIC_BYPASS
1223	diag = ixgbe_bypass_init_hw(hw);
1224#else
1225	diag = ixgbe_init_hw(hw);
1226#endif /* RTE_NIC_BYPASS */
1227
1228	/*
1229	 * Devices with copper phys will fail to initialise if ixgbe_init_hw()
1230	 * is called too soon after the kernel driver unbinding/binding occurs.
1231	 * The failure occurs in ixgbe_identify_phy_generic() for all devices,
1232	 * but for non-copper devies, ixgbe_identify_sfp_module_generic() is
1233	 * also called. See ixgbe_identify_phy_82599(). The reason for the
1234	 * failure is not known, and only occuts when virtualisation features
1235	 * are disabled in the bios. A delay of 100ms  was found to be enough by
1236	 * trial-and-error, and is doubled to be safe.
1237	 */
1238	if (diag && (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper)) {
1239		rte_delay_ms(200);
1240		diag = ixgbe_init_hw(hw);
1241	}
1242
1243	if (diag == IXGBE_ERR_SFP_NOT_PRESENT)
1244		diag = IXGBE_SUCCESS;
1245
1246	if (diag == IXGBE_ERR_EEPROM_VERSION) {
1247		PMD_INIT_LOG(ERR, "This device is a pre-production adapter/"
1248			     "LOM.  Please be aware there may be issues associated "
1249			     "with your hardware.");
1250		PMD_INIT_LOG(ERR, "If you are experiencing problems "
1251			     "please contact your Intel or hardware representative "
1252			     "who provided you with this hardware.");
1253	} else if (diag == IXGBE_ERR_SFP_NOT_SUPPORTED)
1254		PMD_INIT_LOG(ERR, "Unsupported SFP+ Module");
1255	if (diag) {
1256		PMD_INIT_LOG(ERR, "Hardware Initialization Failure: %d", diag);
1257		return -EIO;
1258	}
1259
1260	/* Reset the hw statistics */
1261	ixgbe_dev_stats_reset(eth_dev);
1262
1263	/* disable interrupt */
1264	ixgbe_disable_intr(hw);
1265
1266	/* reset mappings for queue statistics hw counters*/
1267	ixgbe_reset_qstat_mappings(hw);
1268
1269	/* Allocate memory for storing MAC addresses */
1270	eth_dev->data->mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
1271					       hw->mac.num_rar_entries, 0);
1272	if (eth_dev->data->mac_addrs == NULL) {
1273		PMD_INIT_LOG(ERR,
1274			     "Failed to allocate %u bytes needed to store "
1275			     "MAC addresses",
1276			     ETHER_ADDR_LEN * hw->mac.num_rar_entries);
1277		return -ENOMEM;
1278	}
1279	/* Copy the permanent MAC address */
1280	ether_addr_copy((struct ether_addr *) hw->mac.perm_addr,
1281			&eth_dev->data->mac_addrs[0]);
1282
1283	/* Allocate memory for storing hash filter MAC addresses */
1284	eth_dev->data->hash_mac_addrs = rte_zmalloc("ixgbe", ETHER_ADDR_LEN *
1285						    IXGBE_VMDQ_NUM_UC_MAC, 0);
1286	if (eth_dev->data->hash_mac_addrs == NULL) {
1287		PMD_INIT_LOG(ERR,
1288			     "Failed to allocate %d bytes needed to store MAC addresses",
1289			     ETHER_ADDR_LEN * IXGBE_VMDQ_NUM_UC_MAC);
1290		return -ENOMEM;
1291	}
1292
1293	/* initialize the vfta */
1294	memset(shadow_vfta, 0, sizeof(*shadow_vfta));
1295
1296	/* initialize the hw strip bitmap*/
1297	memset(hwstrip, 0, sizeof(*hwstrip));
1298
1299	/* initialize PF if max_vfs not zero */
1300	ixgbe_pf_host_init(eth_dev);
1301
1302	ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
1303	/* let hardware know driver is loaded */
1304	ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD;
1305	/* Set PF Reset Done bit so PF/VF Mail Ops can work */
1306	ctrl_ext |= IXGBE_CTRL_EXT_PFRSTD;
1307	IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext);
1308	IXGBE_WRITE_FLUSH(hw);
1309
1310	if (ixgbe_is_sfp(hw) && hw->phy.sfp_type != ixgbe_sfp_type_not_present)
1311		PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d, SFP+: %d",
1312			     (int) hw->mac.type, (int) hw->phy.type,
1313			     (int) hw->phy.sfp_type);
1314	else
1315		PMD_INIT_LOG(DEBUG, "MAC: %d, PHY: %d",
1316			     (int) hw->mac.type, (int) hw->phy.type);
1317
1318	PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x",
1319		     eth_dev->data->port_id, pci_dev->id.vendor_id,
1320		     pci_dev->id.device_id);
1321
1322	rte_intr_callback_register(intr_handle,
1323				   ixgbe_dev_interrupt_handler, eth_dev);
1324
1325	/* enable uio/vfio intr/eventfd mapping */
1326	rte_intr_enable(intr_handle);
1327
1328	/* enable support intr */
1329	ixgbe_enable_intr(eth_dev);
1330
1331	/* initialize filter info */
1332	memset(filter_info, 0,
1333	       sizeof(struct ixgbe_filter_info));
1334
1335	/* initialize 5tuple filter list */
1336	TAILQ_INIT(&filter_info->fivetuple_list);
1337
1338	/* initialize flow director filter list & hash */
1339	ixgbe_fdir_filter_init(eth_dev);
1340
1341	/* initialize l2 tunnel filter list & hash */
1342	ixgbe_l2_tn_filter_init(eth_dev);
1343
1344	TAILQ_INIT(&filter_ntuple_list);
1345	TAILQ_INIT(&filter_ethertype_list);
1346	TAILQ_INIT(&filter_syn_list);
1347	TAILQ_INIT(&filter_fdir_list);
1348	TAILQ_INIT(&filter_l2_tunnel_list);
1349	TAILQ_INIT(&ixgbe_flow_list);
1350
1351	return 0;
1352}
1353
1354static int
1355eth_ixgbe_dev_uninit(struct rte_eth_dev *eth_dev)
1356{
1357	struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(eth_dev);
1358	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
1359	struct ixgbe_hw *hw;
1360
1361	PMD_INIT_FUNC_TRACE();
1362
1363	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1364		return -EPERM;
1365
1366	hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1367
1368	if (hw->adapter_stopped == 0)
1369		ixgbe_dev_close(eth_dev);
1370
1371	eth_dev->dev_ops = NULL;
1372	eth_dev->rx_pkt_burst = NULL;
1373	eth_dev->tx_pkt_burst = NULL;
1374
1375	/* Unlock any pending hardware semaphore */
1376	ixgbe_swfw_lock_reset(hw);
1377
1378	/* disable uio intr before callback unregister */
1379	rte_intr_disable(intr_handle);
1380	rte_intr_callback_unregister(intr_handle,
1381				     ixgbe_dev_interrupt_handler, eth_dev);
1382
1383	/* uninitialize PF if max_vfs not zero */
1384	ixgbe_pf_host_uninit(eth_dev);
1385
1386	rte_free(eth_dev->data->mac_addrs);
1387	eth_dev->data->mac_addrs = NULL;
1388
1389	rte_free(eth_dev->data->hash_mac_addrs);
1390	eth_dev->data->hash_mac_addrs = NULL;
1391
1392	/* remove all the fdir filters & hash */
1393	ixgbe_fdir_filter_uninit(eth_dev);
1394
1395	/* remove all the L2 tunnel filters & hash */
1396	ixgbe_l2_tn_filter_uninit(eth_dev);
1397
1398	/* Remove all ntuple filters of the device */
1399	ixgbe_ntuple_filter_uninit(eth_dev);
1400
1401	/* clear all the filters list */
1402	ixgbe_filterlist_flush();
1403
1404	return 0;
1405}
1406
1407static int ixgbe_ntuple_filter_uninit(struct rte_eth_dev *eth_dev)
1408{
1409	struct ixgbe_filter_info *filter_info =
1410		IXGBE_DEV_PRIVATE_TO_FILTER_INFO(eth_dev->data->dev_private);
1411	struct ixgbe_5tuple_filter *p_5tuple;
1412
1413	while ((p_5tuple = TAILQ_FIRST(&filter_info->fivetuple_list))) {
1414		TAILQ_REMOVE(&filter_info->fivetuple_list,
1415			     p_5tuple,
1416			     entries);
1417		rte_free(p_5tuple);
1418	}
1419	memset(filter_info->fivetuple_mask, 0,
1420	       sizeof(uint32_t) * IXGBE_5TUPLE_ARRAY_SIZE);
1421
1422	return 0;
1423}
1424
1425static int ixgbe_fdir_filter_uninit(struct rte_eth_dev *eth_dev)
1426{
1427	struct ixgbe_hw_fdir_info *fdir_info =
1428		IXGBE_DEV_PRIVATE_TO_FDIR_INFO(eth_dev->data->dev_private);
1429	struct ixgbe_fdir_filter *fdir_filter;
1430
1431		if (fdir_info->hash_map)
1432		rte_free(fdir_info->hash_map);
1433	if (fdir_info->hash_handle)
1434		rte_hash_free(fdir_info->hash_handle);
1435
1436	while ((fdir_filter = TAILQ_FIRST(&fdir_info->fdir_list))) {
1437		TAILQ_REMOVE(&fdir_info->fdir_list,
1438			     fdir_filter,
1439			     entries);
1440		rte_free(fdir_filter);
1441	}
1442
1443	return 0;
1444}
1445
1446static int ixgbe_l2_tn_filter_uninit(struct rte_eth_dev *eth_dev)
1447{
1448	struct ixgbe_l2_tn_info *l2_tn_info =
1449		IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(eth_dev->data->dev_private);
1450	struct ixgbe_l2_tn_filter *l2_tn_filter;
1451
1452	if (l2_tn_info->hash_map)
1453		rte_free(l2_tn_info->hash_map);
1454	if (l2_tn_info->hash_handle)
1455		rte_hash_free(l2_tn_info->hash_handle);
1456
1457	while ((l2_tn_filter = TAILQ_FIRST(&l2_tn_info->l2_tn_list))) {
1458		TAILQ_REMOVE(&l2_tn_info->l2_tn_list,
1459			     l2_tn_filter,
1460			     entries);
1461		rte_free(l2_tn_filter);
1462	}
1463
1464	return 0;
1465}
1466
1467static int ixgbe_fdir_filter_init(struct rte_eth_dev *eth_dev)
1468{
1469	struct ixgbe_hw_fdir_info *fdir_info =
1470		IXGBE_DEV_PRIVATE_TO_FDIR_INFO(eth_dev->data->dev_private);
1471	char fdir_hash_name[RTE_HASH_NAMESIZE];
1472	struct rte_hash_parameters fdir_hash_params = {
1473		.name = fdir_hash_name,
1474		.entries = IXGBE_MAX_FDIR_FILTER_NUM,
1475		.key_len = sizeof(union ixgbe_atr_input),
1476		.hash_func = rte_hash_crc,
1477		.hash_func_init_val = 0,
1478		.socket_id = rte_socket_id(),
1479	};
1480
1481	TAILQ_INIT(&fdir_info->fdir_list);
1482	snprintf(fdir_hash_name, RTE_HASH_NAMESIZE,
1483		 "fdir_%s", eth_dev->data->name);
1484	fdir_info->hash_handle = rte_hash_create(&fdir_hash_params);
1485	if (!fdir_info->hash_handle) {
1486		PMD_INIT_LOG(ERR, "Failed to create fdir hash table!");
1487		return -EINVAL;
1488	}
1489	fdir_info->hash_map = rte_zmalloc("ixgbe",
1490					  sizeof(struct ixgbe_fdir_filter *) *
1491					  IXGBE_MAX_FDIR_FILTER_NUM,
1492					  0);
1493	if (!fdir_info->hash_map) {
1494		PMD_INIT_LOG(ERR,
1495			     "Failed to allocate memory for fdir hash map!");
1496		return -ENOMEM;
1497	}
1498	fdir_info->mask_added = FALSE;
1499
1500	return 0;
1501}
1502
1503static int ixgbe_l2_tn_filter_init(struct rte_eth_dev *eth_dev)
1504{
1505	struct ixgbe_l2_tn_info *l2_tn_info =
1506		IXGBE_DEV_PRIVATE_TO_L2_TN_INFO(eth_dev->data->dev_private);
1507	char l2_tn_hash_name[RTE_HASH_NAMESIZE];
1508	struct rte_hash_parameters l2_tn_hash_params = {
1509		.name = l2_tn_hash_name,
1510		.entries = IXGBE_MAX_L2_TN_FILTER_NUM,
1511		.key_len = sizeof(struct ixgbe_l2_tn_key),
1512		.hash_func = rte_hash_crc,
1513		.hash_func_init_val = 0,
1514		.socket_id = rte_socket_id(),
1515	};
1516
1517	TAILQ_INIT(&l2_tn_info->l2_tn_list);
1518	snprintf(l2_tn_hash_name, RTE_HASH_NAMESIZE,
1519		 "l2_tn_%s", eth_dev->data->name);
1520	l2_tn_info->hash_handle = rte_hash_create(&l2_tn_hash_params);
1521	if (!l2_tn_info->hash_handle) {
1522		PMD_INIT_LOG(ERR, "Failed to create L2 TN hash table!");
1523		return -EINVAL;
1524	}
1525	l2_tn_info->hash_map = rte_zmalloc("ixgbe",
1526				   sizeof(struct ixgbe_l2_tn_filter *) *
1527				   IXGBE_MAX_L2_TN_FILTER_NUM,
1528				   0);
1529	if (!l2_tn_info->hash_map) {
1530		PMD_INIT_LOG(ERR,
1531			"Failed to allocate memory for L2 TN hash map!");
1532		return -ENOMEM;
1533	}
1534	l2_tn_info->e_tag_en = FALSE;
1535	l2_tn_info->e_tag_fwd_en = FALSE;
1536	l2_tn_info->e_tag_ether_type = DEFAULT_ETAG_ETYPE;
1537
1538	return 0;
1539}
1540/*
1541 * Negotiate mailbox API version with the PF.
1542 * After reset API version is always set to the basic one (ixgbe_mbox_api_10).
1543 * Then we try to negotiate starting with the most recent one.
1544 * If all negotiation attempts fail, then we will proceed with
1545 * the default one (ixgbe_mbox_api_10).
1546 */
1547static void
1548ixgbevf_negotiate_api(struct ixgbe_hw *hw)
1549{
1550	int32_t i;
1551
1552	/* start with highest supported, proceed down */
1553	static const enum ixgbe_pfvf_api_rev sup_ver[] = {
1554		ixgbe_mbox_api_12,
1555		ixgbe_mbox_api_11,
1556		ixgbe_mbox_api_10,
1557	};
1558
1559	for (i = 0;
1560			i != RTE_DIM(sup_ver) &&
1561			ixgbevf_negotiate_api_version(hw, sup_ver[i]) != 0;
1562			i++)
1563		;
1564}
1565
1566static void
1567generate_random_mac_addr(struct ether_addr *mac_addr)
1568{
1569	uint64_t random;
1570
1571	/* Set Organizationally Unique Identifier (OUI) prefix. */
1572	mac_addr->addr_bytes[0] = 0x00;
1573	mac_addr->addr_bytes[1] = 0x09;
1574	mac_addr->addr_bytes[2] = 0xC0;
1575	/* Force indication of locally assigned MAC address. */
1576	mac_addr->addr_bytes[0] |= ETHER_LOCAL_ADMIN_ADDR;
1577	/* Generate the last 3 bytes of the MAC address with a random number. */
1578	random = rte_rand();
1579	memcpy(&mac_addr->addr_bytes[3], &random, 3);
1580}
1581
1582/*
1583 * Virtual Function device init
1584 */
1585static int
1586eth_ixgbevf_dev_init(struct rte_eth_dev *eth_dev)
1587{
1588	int diag;
1589	uint32_t tc, tcs;
1590	struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(eth_dev);
1591	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
1592	struct ixgbe_hw *hw =
1593		IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1594	struct ixgbe_vfta *shadow_vfta =
1595		IXGBE_DEV_PRIVATE_TO_VFTA(eth_dev->data->dev_private);
1596	struct ixgbe_hwstrip *hwstrip =
1597		IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(eth_dev->data->dev_private);
1598	struct ether_addr *perm_addr = (struct ether_addr *) hw->mac.perm_addr;
1599
1600	PMD_INIT_FUNC_TRACE();
1601
1602	eth_dev->dev_ops = &ixgbevf_eth_dev_ops;
1603	eth_dev->rx_pkt_burst = &ixgbe_recv_pkts;
1604	eth_dev->tx_pkt_burst = &ixgbe_xmit_pkts;
1605
1606	/* for secondary processes, we don't initialise any further as primary
1607	 * has already done this work. Only check we don't need a different
1608	 * RX function
1609	 */
1610	if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
1611		struct ixgbe_tx_queue *txq;
1612		/* TX queue function in primary, set by last queue initialized
1613		 * Tx queue may not initialized by primary process
1614		 */
1615		if (eth_dev->data->tx_queues) {
1616			txq = eth_dev->data->tx_queues[eth_dev->data->nb_tx_queues - 1];
1617			ixgbe_set_tx_function(eth_dev, txq);
1618		} else {
1619			/* Use default TX function if we get here */
1620			PMD_INIT_LOG(NOTICE,
1621				     "No TX queues configured yet. Using default TX function.");
1622		}
1623
1624		ixgbe_set_rx_function(eth_dev);
1625
1626		return 0;
1627	}
1628
1629	rte_eth_copy_pci_info(eth_dev, pci_dev);
1630	eth_dev->data->dev_flags |= RTE_ETH_DEV_DETACHABLE;
1631
1632	hw->device_id = pci_dev->id.device_id;
1633	hw->vendor_id = pci_dev->id.vendor_id;
1634	hw->hw_addr = (void *)pci_dev->mem_resource[0].addr;
1635
1636	/* initialize the vfta */
1637	memset(shadow_vfta, 0, sizeof(*shadow_vfta));
1638
1639	/* initialize the hw strip bitmap*/
1640	memset(hwstrip, 0, sizeof(*hwstrip));
1641
1642	/* Initialize the shared code (base driver) */
1643	diag = ixgbe_init_shared_code(hw);
1644	if (diag != IXGBE_SUCCESS) {
1645		PMD_INIT_LOG(ERR, "Shared code init failed for ixgbevf: %d", diag);
1646		return -EIO;
1647	}
1648
1649	/* init_mailbox_params */
1650	hw->mbx.ops.init_params(hw);
1651
1652	/* Reset the hw statistics */
1653	ixgbevf_dev_stats_reset(eth_dev);
1654
1655	/* Disable the interrupts for VF */
1656	ixgbevf_intr_disable(hw);
1657
1658	hw->mac.num_rar_entries = 128; /* The MAX of the underlying PF */
1659	diag = hw->mac.ops.reset_hw(hw);
1660
1661	/*
1662	 * The VF reset operation returns the IXGBE_ERR_INVALID_MAC_ADDR when
1663	 * the underlying PF driver has not assigned a MAC address to the VF.
1664	 * In this case, assign a random MAC address.
1665	 */
1666	if ((diag != IXGBE_SUCCESS) && (diag != IXGBE_ERR_INVALID_MAC_ADDR)) {
1667		PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
1668		return diag;
1669	}
1670
1671	/* negotiate mailbox API version to use with the PF. */
1672	ixgbevf_negotiate_api(hw);
1673
1674	/* Get Rx/Tx queue count via mailbox, which is ready after reset_hw */
1675	ixgbevf_get_queues(hw, &tcs, &tc);
1676
1677	/* Allocate memory for storing MAC addresses */
1678	eth_dev->data->mac_addrs = rte_zmalloc("ixgbevf", ETHER_ADDR_LEN *
1679					       hw->mac.num_rar_entries, 0);
1680	if (eth_dev->data->mac_addrs == NULL) {
1681		PMD_INIT_LOG(ERR,
1682			     "Failed to allocate %u bytes needed to store "
1683			     "MAC addresses",
1684			     ETHER_ADDR_LEN * hw->mac.num_rar_entries);
1685		return -ENOMEM;
1686	}
1687
1688	/* Generate a random MAC address, if none was assigned by PF. */
1689	if (is_zero_ether_addr(perm_addr)) {
1690		generate_random_mac_addr(perm_addr);
1691		diag = ixgbe_set_rar_vf(hw, 1, perm_addr->addr_bytes, 0, 1);
1692		if (diag) {
1693			rte_free(eth_dev->data->mac_addrs);
1694			eth_dev->data->mac_addrs = NULL;
1695			return diag;
1696		}
1697		PMD_INIT_LOG(INFO, "\tVF MAC address not assigned by Host PF");
1698		PMD_INIT_LOG(INFO, "\tAssign randomly generated MAC address "
1699			     "%02x:%02x:%02x:%02x:%02x:%02x",
1700			     perm_addr->addr_bytes[0],
1701			     perm_addr->addr_bytes[1],
1702			     perm_addr->addr_bytes[2],
1703			     perm_addr->addr_bytes[3],
1704			     perm_addr->addr_bytes[4],
1705			     perm_addr->addr_bytes[5]);
1706	}
1707
1708	/* Copy the permanent MAC address */
1709	ether_addr_copy(perm_addr, &eth_dev->data->mac_addrs[0]);
1710
1711	/* reset the hardware with the new settings */
1712	diag = hw->mac.ops.start_hw(hw);
1713	switch (diag) {
1714	case  0:
1715		break;
1716
1717	default:
1718		PMD_INIT_LOG(ERR, "VF Initialization Failure: %d", diag);
1719		return -EIO;
1720	}
1721
1722	rte_intr_callback_register(intr_handle,
1723				   ixgbevf_dev_interrupt_handler, eth_dev);
1724	rte_intr_enable(intr_handle);
1725	ixgbevf_intr_enable(hw);
1726
1727	PMD_INIT_LOG(DEBUG, "port %d vendorID=0x%x deviceID=0x%x mac.type=%s",
1728		     eth_dev->data->port_id, pci_dev->id.vendor_id,
1729		     pci_dev->id.device_id, "ixgbe_mac_82599_vf");
1730
1731	return 0;
1732}
1733
1734/* Virtual Function device uninit */
1735
1736static int
1737eth_ixgbevf_dev_uninit(struct rte_eth_dev *eth_dev)
1738{
1739	struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(eth_dev);
1740	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
1741	struct ixgbe_hw *hw;
1742
1743	PMD_INIT_FUNC_TRACE();
1744
1745	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1746		return -EPERM;
1747
1748	hw = IXGBE_DEV_PRIVATE_TO_HW(eth_dev->data->dev_private);
1749
1750	if (hw->adapter_stopped == 0)
1751		ixgbevf_dev_close(eth_dev);
1752
1753	eth_dev->dev_ops = NULL;
1754	eth_dev->rx_pkt_burst = NULL;
1755	eth_dev->tx_pkt_burst = NULL;
1756
1757	/* Disable the interrupts for VF */
1758	ixgbevf_intr_disable(hw);
1759
1760	rte_free(eth_dev->data->mac_addrs);
1761	eth_dev->data->mac_addrs = NULL;
1762
1763	rte_intr_disable(intr_handle);
1764	rte_intr_callback_unregister(intr_handle,
1765				     ixgbevf_dev_interrupt_handler, eth_dev);
1766
1767	return 0;
1768}
1769
1770static struct eth_driver rte_ixgbe_pmd = {
1771	.pci_drv = {
1772		.id_table = pci_id_ixgbe_map,
1773		.drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
1774		.probe = rte_eth_dev_pci_probe,
1775		.remove = rte_eth_dev_pci_remove,
1776	},
1777	.eth_dev_init = eth_ixgbe_dev_init,
1778	.eth_dev_uninit = eth_ixgbe_dev_uninit,
1779	.dev_private_size = sizeof(struct ixgbe_adapter),
1780};
1781
1782/*
1783 * virtual function driver struct
1784 */
1785static struct eth_driver rte_ixgbevf_pmd = {
1786	.pci_drv = {
1787		.id_table = pci_id_ixgbevf_map,
1788		.drv_flags = RTE_PCI_DRV_NEED_MAPPING,
1789		.probe = rte_eth_dev_pci_probe,
1790		.remove = rte_eth_dev_pci_remove,
1791	},
1792	.eth_dev_init = eth_ixgbevf_dev_init,
1793	.eth_dev_uninit = eth_ixgbevf_dev_uninit,
1794	.dev_private_size = sizeof(struct ixgbe_adapter),
1795};
1796
1797static int
1798ixgbe_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1799{
1800	struct ixgbe_hw *hw =
1801		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1802	struct ixgbe_vfta *shadow_vfta =
1803		IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1804	uint32_t vfta;
1805	uint32_t vid_idx;
1806	uint32_t vid_bit;
1807
1808	vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
1809	vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
1810	vfta = IXGBE_READ_REG(hw, IXGBE_VFTA(vid_idx));
1811	if (on)
1812		vfta |= vid_bit;
1813	else
1814		vfta &= ~vid_bit;
1815	IXGBE_WRITE_REG(hw, IXGBE_VFTA(vid_idx), vfta);
1816
1817	/* update local VFTA copy */
1818	shadow_vfta->vfta[vid_idx] = vfta;
1819
1820	return 0;
1821}
1822
1823static void
1824ixgbe_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
1825{
1826	if (on)
1827		ixgbe_vlan_hw_strip_enable(dev, queue);
1828	else
1829		ixgbe_vlan_hw_strip_disable(dev, queue);
1830}
1831
1832static int
1833ixgbe_vlan_tpid_set(struct rte_eth_dev *dev,
1834		    enum rte_vlan_type vlan_type,
1835		    uint16_t tpid)
1836{
1837	struct ixgbe_hw *hw =
1838		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1839	int ret = 0;
1840	uint32_t reg;
1841	uint32_t qinq;
1842
1843	qinq = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1844	qinq &= IXGBE_DMATXCTL_GDV;
1845
1846	switch (vlan_type) {
1847	case ETH_VLAN_TYPE_INNER:
1848		if (qinq) {
1849			reg = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1850			reg = (reg & (~IXGBE_VLNCTRL_VET)) | (uint32_t)tpid;
1851			IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, reg);
1852			reg = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1853			reg = (reg & (~IXGBE_DMATXCTL_VT_MASK))
1854				| ((uint32_t)tpid << IXGBE_DMATXCTL_VT_SHIFT);
1855			IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg);
1856		} else {
1857			ret = -ENOTSUP;
1858			PMD_DRV_LOG(ERR, "Inner type is not supported"
1859				    " by single VLAN");
1860		}
1861		break;
1862	case ETH_VLAN_TYPE_OUTER:
1863		if (qinq) {
1864			/* Only the high 16-bits is valid */
1865			IXGBE_WRITE_REG(hw, IXGBE_EXVET, (uint32_t)tpid <<
1866					IXGBE_EXVET_VET_EXT_SHIFT);
1867		} else {
1868			reg = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1869			reg = (reg & (~IXGBE_VLNCTRL_VET)) | (uint32_t)tpid;
1870			IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, reg);
1871			reg = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
1872			reg = (reg & (~IXGBE_DMATXCTL_VT_MASK))
1873				| ((uint32_t)tpid << IXGBE_DMATXCTL_VT_SHIFT);
1874			IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg);
1875		}
1876
1877		break;
1878	default:
1879		ret = -EINVAL;
1880		PMD_DRV_LOG(ERR, "Unsupported VLAN type %d", vlan_type);
1881		break;
1882	}
1883
1884	return ret;
1885}
1886
1887void
1888ixgbe_vlan_hw_filter_disable(struct rte_eth_dev *dev)
1889{
1890	struct ixgbe_hw *hw =
1891		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1892	uint32_t vlnctrl;
1893
1894	PMD_INIT_FUNC_TRACE();
1895
1896	/* Filter Table Disable */
1897	vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1898	vlnctrl &= ~IXGBE_VLNCTRL_VFE;
1899
1900	IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
1901}
1902
1903void
1904ixgbe_vlan_hw_filter_enable(struct rte_eth_dev *dev)
1905{
1906	struct ixgbe_hw *hw =
1907		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1908	struct ixgbe_vfta *shadow_vfta =
1909		IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
1910	uint32_t vlnctrl;
1911	uint16_t i;
1912
1913	PMD_INIT_FUNC_TRACE();
1914
1915	/* Filter Table Enable */
1916	vlnctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
1917	vlnctrl &= ~IXGBE_VLNCTRL_CFIEN;
1918	vlnctrl |= IXGBE_VLNCTRL_VFE;
1919
1920	IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl);
1921
1922	/* write whatever is in local vfta copy */
1923	for (i = 0; i < IXGBE_VFTA_SIZE; i++)
1924		IXGBE_WRITE_REG(hw, IXGBE_VFTA(i), shadow_vfta->vfta[i]);
1925}
1926
1927static void
1928ixgbe_vlan_hw_strip_bitmap_set(struct rte_eth_dev *dev, uint16_t queue, bool on)
1929{
1930	struct ixgbe_hwstrip *hwstrip =
1931		IXGBE_DEV_PRIVATE_TO_HWSTRIP_BITMAP(dev->data->dev_private);
1932	struct ixgbe_rx_queue *rxq;
1933
1934	if (queue >= IXGBE_MAX_RX_QUEUE_NUM)
1935		return;
1936
1937	if (on)
1938		IXGBE_SET_HWSTRIP(hwstrip, queue);
1939	else
1940		IXGBE_CLEAR_HWSTRIP(hwstrip, queue);
1941
1942	if (queue >= dev->data->nb_rx_queues)
1943		return;
1944
1945	rxq = dev->data->rx_queues[queue];
1946
1947	if (on)
1948		rxq->vlan_flags = PKT_RX_VLAN_PKT | PKT_RX_VLAN_STRIPPED;
1949	else
1950		rxq->vlan_flags = PKT_RX_VLAN_PKT;
1951}
1952
1953static void
1954ixgbe_vlan_hw_strip_disable(struct rte_eth_dev *dev, uint16_t queue)
1955{
1956	struct ixgbe_hw *hw =
1957		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1958	uint32_t ctrl;
1959
1960	PMD_INIT_FUNC_TRACE();
1961
1962	if (hw->mac.type == ixgbe_mac_82598EB) {
1963		/* No queue level support */
1964		PMD_INIT_LOG(NOTICE, "82598EB not support queue level hw strip");
1965		return;
1966	}
1967
1968	/* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1969	ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
1970	ctrl &= ~IXGBE_RXDCTL_VME;
1971	IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
1972
1973	/* record those setting for HW strip per queue */
1974	ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 0);
1975}
1976
1977static void
1978ixgbe_vlan_hw_strip_enable(struct rte_eth_dev *dev, uint16_t queue)
1979{
1980	struct ixgbe_hw *hw =
1981		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
1982	uint32_t ctrl;
1983
1984	PMD_INIT_FUNC_TRACE();
1985
1986	if (hw->mac.type == ixgbe_mac_82598EB) {
1987		/* No queue level supported */
1988		PMD_INIT_LOG(NOTICE, "82598EB not support queue level hw strip");
1989		return;
1990	}
1991
1992	/* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
1993	ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
1994	ctrl |= IXGBE_RXDCTL_VME;
1995	IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
1996
1997	/* record those setting for HW strip per queue */
1998	ixgbe_vlan_hw_strip_bitmap_set(dev, queue, 1);
1999}
2000
2001void
2002ixgbe_vlan_hw_strip_disable_all(struct rte_eth_dev *dev)
2003{
2004	struct ixgbe_hw *hw =
2005		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2006	uint32_t ctrl;
2007	uint16_t i;
2008	struct ixgbe_rx_queue *rxq;
2009
2010	PMD_INIT_FUNC_TRACE();
2011
2012	if (hw->mac.type == ixgbe_mac_82598EB) {
2013		ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
2014		ctrl &= ~IXGBE_VLNCTRL_VME;
2015		IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
2016	} else {
2017		/* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
2018		for (i = 0; i < dev->data->nb_rx_queues; i++) {
2019			rxq = dev->data->rx_queues[i];
2020			ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
2021			ctrl &= ~IXGBE_RXDCTL_VME;
2022			IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxq->reg_idx), ctrl);
2023
2024			/* record those setting for HW strip per queue */
2025			ixgbe_vlan_hw_strip_bitmap_set(dev, i, 0);
2026		}
2027	}
2028}
2029
2030void
2031ixgbe_vlan_hw_strip_enable_all(struct rte_eth_dev *dev)
2032{
2033	struct ixgbe_hw *hw =
2034		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2035	uint32_t ctrl;
2036	uint16_t i;
2037	struct ixgbe_rx_queue *rxq;
2038
2039	PMD_INIT_FUNC_TRACE();
2040
2041	if (hw->mac.type == ixgbe_mac_82598EB) {
2042		ctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
2043		ctrl |= IXGBE_VLNCTRL_VME;
2044		IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, ctrl);
2045	} else {
2046		/* Other 10G NIC, the VLAN strip can be setup per queue in RXDCTL */
2047		for (i = 0; i < dev->data->nb_rx_queues; i++) {
2048			rxq = dev->data->rx_queues[i];
2049			ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rxq->reg_idx));
2050			ctrl |= IXGBE_RXDCTL_VME;
2051			IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rxq->reg_idx), ctrl);
2052
2053			/* record those setting for HW strip per queue */
2054			ixgbe_vlan_hw_strip_bitmap_set(dev, i, 1);
2055		}
2056	}
2057}
2058
2059static void
2060ixgbe_vlan_hw_extend_disable(struct rte_eth_dev *dev)
2061{
2062	struct ixgbe_hw *hw =
2063		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2064	uint32_t ctrl;
2065
2066	PMD_INIT_FUNC_TRACE();
2067
2068	/* DMATXCTRL: Geric Double VLAN Disable */
2069	ctrl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2070	ctrl &= ~IXGBE_DMATXCTL_GDV;
2071	IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
2072
2073	/* CTRL_EXT: Global Double VLAN Disable */
2074	ctrl = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
2075	ctrl &= ~IXGBE_EXTENDED_VLAN;
2076	IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
2077
2078}
2079
2080static void
2081ixgbe_vlan_hw_extend_enable(struct rte_eth_dev *dev)
2082{
2083	struct ixgbe_hw *hw =
2084		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2085	uint32_t ctrl;
2086
2087	PMD_INIT_FUNC_TRACE();
2088
2089	/* DMATXCTRL: Geric Double VLAN Enable */
2090	ctrl  = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2091	ctrl |= IXGBE_DMATXCTL_GDV;
2092	IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, ctrl);
2093
2094	/* CTRL_EXT: Global Double VLAN Enable */
2095	ctrl  = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT);
2096	ctrl |= IXGBE_EXTENDED_VLAN;
2097	IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl);
2098
2099	/* Clear pooling mode of PFVTCTL. It's required by X550. */
2100	if (hw->mac.type == ixgbe_mac_X550 ||
2101	    hw->mac.type == ixgbe_mac_X550EM_x ||
2102	    hw->mac.type == ixgbe_mac_X550EM_a) {
2103		ctrl = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
2104		ctrl &= ~IXGBE_VT_CTL_POOLING_MODE_MASK;
2105		IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, ctrl);
2106	}
2107
2108	/*
2109	 * VET EXT field in the EXVET register = 0x8100 by default
2110	 * So no need to change. Same to VT field of DMATXCTL register
2111	 */
2112}
2113
2114static void
2115ixgbe_vlan_offload_set(struct rte_eth_dev *dev, int mask)
2116{
2117	if (mask & ETH_VLAN_STRIP_MASK) {
2118		if (dev->data->dev_conf.rxmode.hw_vlan_strip)
2119			ixgbe_vlan_hw_strip_enable_all(dev);
2120		else
2121			ixgbe_vlan_hw_strip_disable_all(dev);
2122	}
2123
2124	if (mask & ETH_VLAN_FILTER_MASK) {
2125		if (dev->data->dev_conf.rxmode.hw_vlan_filter)
2126			ixgbe_vlan_hw_filter_enable(dev);
2127		else
2128			ixgbe_vlan_hw_filter_disable(dev);
2129	}
2130
2131	if (mask & ETH_VLAN_EXTEND_MASK) {
2132		if (dev->data->dev_conf.rxmode.hw_vlan_extend)
2133			ixgbe_vlan_hw_extend_enable(dev);
2134		else
2135			ixgbe_vlan_hw_extend_disable(dev);
2136	}
2137}
2138
2139static void
2140ixgbe_vmdq_vlan_hw_filter_enable(struct rte_eth_dev *dev)
2141{
2142	struct ixgbe_hw *hw =
2143		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2144	/* VLNCTRL: enable vlan filtering and allow all vlan tags through */
2145	uint32_t vlanctrl = IXGBE_READ_REG(hw, IXGBE_VLNCTRL);
2146
2147	vlanctrl |= IXGBE_VLNCTRL_VFE; /* enable vlan filters */
2148	IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlanctrl);
2149}
2150
2151static int
2152ixgbe_check_vf_rss_rxq_num(struct rte_eth_dev *dev, uint16_t nb_rx_q)
2153{
2154	struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(dev);
2155
2156	switch (nb_rx_q) {
2157	case 1:
2158	case 2:
2159		RTE_ETH_DEV_SRIOV(dev).active = ETH_64_POOLS;
2160		break;
2161	case 4:
2162		RTE_ETH_DEV_SRIOV(dev).active = ETH_32_POOLS;
2163		break;
2164	default:
2165		return -EINVAL;
2166	}
2167
2168	RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = nb_rx_q;
2169	RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx = pci_dev->max_vfs * nb_rx_q;
2170
2171	return 0;
2172}
2173
2174static int
2175ixgbe_check_mq_mode(struct rte_eth_dev *dev)
2176{
2177	struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
2178	struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2179	uint16_t nb_rx_q = dev->data->nb_rx_queues;
2180	uint16_t nb_tx_q = dev->data->nb_tx_queues;
2181
2182	if (RTE_ETH_DEV_SRIOV(dev).active != 0) {
2183		/* check multi-queue mode */
2184		switch (dev_conf->rxmode.mq_mode) {
2185		case ETH_MQ_RX_VMDQ_DCB:
2186			PMD_INIT_LOG(INFO, "ETH_MQ_RX_VMDQ_DCB mode supported in SRIOV");
2187			break;
2188		case ETH_MQ_RX_VMDQ_DCB_RSS:
2189			/* DCB/RSS VMDQ in SRIOV mode, not implement yet */
2190			PMD_INIT_LOG(ERR, "SRIOV active,"
2191					" unsupported mq_mode rx %d.",
2192					dev_conf->rxmode.mq_mode);
2193			return -EINVAL;
2194		case ETH_MQ_RX_RSS:
2195		case ETH_MQ_RX_VMDQ_RSS:
2196			dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_RSS;
2197			if (nb_rx_q <= RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool)
2198				if (ixgbe_check_vf_rss_rxq_num(dev, nb_rx_q)) {
2199					PMD_INIT_LOG(ERR, "SRIOV is active,"
2200						" invalid queue number"
2201						" for VMDQ RSS, allowed"
2202						" value are 1, 2 or 4.");
2203					return -EINVAL;
2204				}
2205			break;
2206		case ETH_MQ_RX_VMDQ_ONLY:
2207		case ETH_MQ_RX_NONE:
2208			/* if nothing mq mode configure, use default scheme */
2209			dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_ONLY;
2210			if (RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool > 1)
2211				RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = 1;
2212			break;
2213		default: /* ETH_MQ_RX_DCB, ETH_MQ_RX_DCB_RSS or ETH_MQ_TX_DCB*/
2214			/* SRIOV only works in VMDq enable mode */
2215			PMD_INIT_LOG(ERR, "SRIOV is active,"
2216					" wrong mq_mode rx %d.",
2217					dev_conf->rxmode.mq_mode);
2218			return -EINVAL;
2219		}
2220
2221		switch (dev_conf->txmode.mq_mode) {
2222		case ETH_MQ_TX_VMDQ_DCB:
2223			PMD_INIT_LOG(INFO, "ETH_MQ_TX_VMDQ_DCB mode supported in SRIOV");
2224			dev->data->dev_conf.txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB;
2225			break;
2226		default: /* ETH_MQ_TX_VMDQ_ONLY or ETH_MQ_TX_NONE */
2227			dev->data->dev_conf.txmode.mq_mode = ETH_MQ_TX_VMDQ_ONLY;
2228			break;
2229		}
2230
2231		/* check valid queue number */
2232		if ((nb_rx_q > RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool) ||
2233		    (nb_tx_q > RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool)) {
2234			PMD_INIT_LOG(ERR, "SRIOV is active,"
2235					" nb_rx_q=%d nb_tx_q=%d queue number"
2236					" must be less than or equal to %d.",
2237					nb_rx_q, nb_tx_q,
2238					RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool);
2239			return -EINVAL;
2240		}
2241	} else {
2242		if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_VMDQ_DCB_RSS) {
2243			PMD_INIT_LOG(ERR, "VMDQ+DCB+RSS mq_mode is"
2244					  " not supported.");
2245			return -EINVAL;
2246		}
2247		/* check configuration for vmdb+dcb mode */
2248		if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_VMDQ_DCB) {
2249			const struct rte_eth_vmdq_dcb_conf *conf;
2250
2251			if (nb_rx_q != IXGBE_VMDQ_DCB_NB_QUEUES) {
2252				PMD_INIT_LOG(ERR, "VMDQ+DCB, nb_rx_q != %d.",
2253						IXGBE_VMDQ_DCB_NB_QUEUES);
2254				return -EINVAL;
2255			}
2256			conf = &dev_conf->rx_adv_conf.vmdq_dcb_conf;
2257			if (!(conf->nb_queue_pools == ETH_16_POOLS ||
2258			       conf->nb_queue_pools == ETH_32_POOLS)) {
2259				PMD_INIT_LOG(ERR, "VMDQ+DCB selected,"
2260						" nb_queue_pools must be %d or %d.",
2261						ETH_16_POOLS, ETH_32_POOLS);
2262				return -EINVAL;
2263			}
2264		}
2265		if (dev_conf->txmode.mq_mode == ETH_MQ_TX_VMDQ_DCB) {
2266			const struct rte_eth_vmdq_dcb_tx_conf *conf;
2267
2268			if (nb_tx_q != IXGBE_VMDQ_DCB_NB_QUEUES) {
2269				PMD_INIT_LOG(ERR, "VMDQ+DCB, nb_tx_q != %d",
2270						 IXGBE_VMDQ_DCB_NB_QUEUES);
2271				return -EINVAL;
2272			}
2273			conf = &dev_conf->tx_adv_conf.vmdq_dcb_tx_conf;
2274			if (!(conf->nb_queue_pools == ETH_16_POOLS ||
2275			       conf->nb_queue_pools == ETH_32_POOLS)) {
2276				PMD_INIT_LOG(ERR, "VMDQ+DCB selected,"
2277						" nb_queue_pools != %d and"
2278						" nb_queue_pools != %d.",
2279						ETH_16_POOLS, ETH_32_POOLS);
2280				return -EINVAL;
2281			}
2282		}
2283
2284		/* For DCB mode check our configuration before we go further */
2285		if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_DCB) {
2286			const struct rte_eth_dcb_rx_conf *conf;
2287
2288			if (nb_rx_q != IXGBE_DCB_NB_QUEUES) {
2289				PMD_INIT_LOG(ERR, "DCB selected, nb_rx_q != %d.",
2290						 IXGBE_DCB_NB_QUEUES);
2291				return -EINVAL;
2292			}
2293			conf = &dev_conf->rx_adv_conf.dcb_rx_conf;
2294			if (!(conf->nb_tcs == ETH_4_TCS ||
2295			       conf->nb_tcs == ETH_8_TCS)) {
2296				PMD_INIT_LOG(ERR, "DCB selected, nb_tcs != %d"
2297						" and nb_tcs != %d.",
2298						ETH_4_TCS, ETH_8_TCS);
2299				return -EINVAL;
2300			}
2301		}
2302
2303		if (dev_conf->txmode.mq_mode == ETH_MQ_TX_DCB) {
2304			const struct rte_eth_dcb_tx_conf *conf;
2305
2306			if (nb_tx_q != IXGBE_DCB_NB_QUEUES) {
2307				PMD_INIT_LOG(ERR, "DCB, nb_tx_q != %d.",
2308						 IXGBE_DCB_NB_QUEUES);
2309				return -EINVAL;
2310			}
2311			conf = &dev_conf->tx_adv_conf.dcb_tx_conf;
2312			if (!(conf->nb_tcs == ETH_4_TCS ||
2313			       conf->nb_tcs == ETH_8_TCS)) {
2314				PMD_INIT_LOG(ERR, "DCB selected, nb_tcs != %d"
2315						" and nb_tcs != %d.",
2316						ETH_4_TCS, ETH_8_TCS);
2317				return -EINVAL;
2318			}
2319		}
2320
2321		/*
2322		 * When DCB/VT is off, maximum number of queues changes,
2323		 * except for 82598EB, which remains constant.
2324		 */
2325		if (dev_conf->txmode.mq_mode == ETH_MQ_TX_NONE &&
2326				hw->mac.type != ixgbe_mac_82598EB) {
2327			if (nb_tx_q > IXGBE_NONE_MODE_TX_NB_QUEUES) {
2328				PMD_INIT_LOG(ERR,
2329					     "Neither VT nor DCB are enabled, "
2330					     "nb_tx_q > %d.",
2331					     IXGBE_NONE_MODE_TX_NB_QUEUES);
2332				return -EINVAL;
2333			}
2334		}
2335	}
2336	return 0;
2337}
2338
2339static int
2340ixgbe_dev_configure(struct rte_eth_dev *dev)
2341{
2342	struct ixgbe_interrupt *intr =
2343		IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2344	struct ixgbe_adapter *adapter =
2345		(struct ixgbe_adapter *)dev->data->dev_private;
2346	int ret;
2347
2348	PMD_INIT_FUNC_TRACE();
2349	/* multipe queue mode checking */
2350	ret  = ixgbe_check_mq_mode(dev);
2351	if (ret != 0) {
2352		PMD_DRV_LOG(ERR, "ixgbe_check_mq_mode fails with %d.",
2353			    ret);
2354		return ret;
2355	}
2356
2357	/* set flag to update link status after init */
2358	intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
2359
2360	/*
2361	 * Initialize to TRUE. If any of Rx queues doesn't meet the bulk
2362	 * allocation or vector Rx preconditions we will reset it.
2363	 */
2364	adapter->rx_bulk_alloc_allowed = true;
2365	adapter->rx_vec_allowed = true;
2366
2367	return 0;
2368}
2369
2370static void
2371ixgbe_dev_phy_intr_setup(struct rte_eth_dev *dev)
2372{
2373	struct ixgbe_hw *hw =
2374		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2375	struct ixgbe_interrupt *intr =
2376		IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
2377	uint32_t gpie;
2378
2379	/* only set up it on X550EM_X */
2380	if (hw->mac.type == ixgbe_mac_X550EM_x) {
2381		gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
2382		gpie |= IXGBE_SDP0_GPIEN_X550EM_x;
2383		IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
2384		if (hw->phy.type == ixgbe_phy_x550em_ext_t)
2385			intr->mask |= IXGBE_EICR_GPI_SDP0_X550EM_x;
2386	}
2387}
2388
2389/*
2390 * Configure device link speed and setup link.
2391 * It returns 0 on success.
2392 */
2393static int
2394ixgbe_dev_start(struct rte_eth_dev *dev)
2395{
2396	struct ixgbe_hw *hw =
2397		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2398	struct ixgbe_vf_info *vfinfo =
2399		*IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private);
2400	struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(dev);
2401	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
2402	uint32_t intr_vector = 0;
2403	int err, link_up = 0, negotiate = 0;
2404	uint32_t speed = 0;
2405	int mask = 0;
2406	int status;
2407	uint16_t vf, idx;
2408	uint32_t *link_speeds;
2409
2410	PMD_INIT_FUNC_TRACE();
2411
2412	/* IXGBE devices don't support:
2413	*    - half duplex (checked afterwards for valid speeds)
2414	*    - fixed speed: TODO implement
2415	*/
2416	if (dev->data->dev_conf.link_speeds & ETH_LINK_SPEED_FIXED) {
2417		PMD_INIT_LOG(ERR, "Invalid link_speeds for port %hhu; fix speed not supported",
2418			     dev->data->port_id);
2419		return -EINVAL;
2420	}
2421
2422	/* disable uio/vfio intr/eventfd mapping */
2423	rte_intr_disable(intr_handle);
2424
2425	/* stop adapter */
2426	hw->adapter_stopped = 0;
2427	ixgbe_stop_adapter(hw);
2428
2429	/* reinitialize adapter
2430	 * this calls reset and start
2431	 */
2432	status = ixgbe_pf_reset_hw(hw);
2433	if (status != 0)
2434		return -1;
2435	hw->mac.ops.start_hw(hw);
2436	hw->mac.get_link_status = true;
2437
2438	/* configure PF module if SRIOV enabled */
2439	ixgbe_pf_host_configure(dev);
2440
2441	ixgbe_dev_phy_intr_setup(dev);
2442
2443	/* check and configure queue intr-vector mapping */
2444	if ((rte_intr_cap_multiple(intr_handle) ||
2445	     !RTE_ETH_DEV_SRIOV(dev).active) &&
2446	    dev->data->dev_conf.intr_conf.rxq != 0) {
2447		intr_vector = dev->data->nb_rx_queues;
2448		if (intr_vector > IXGBE_MAX_INTR_QUEUE_NUM) {
2449			PMD_INIT_LOG(ERR, "At most %d intr queues supported",
2450					IXGBE_MAX_INTR_QUEUE_NUM);
2451			return -ENOTSUP;
2452		}
2453		if (rte_intr_efd_enable(intr_handle, intr_vector))
2454			return -1;
2455	}
2456
2457	if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
2458		intr_handle->intr_vec =
2459			rte_zmalloc("intr_vec",
2460				    dev->data->nb_rx_queues * sizeof(int), 0);
2461		if (intr_handle->intr_vec == NULL) {
2462			PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
2463				     " intr_vec", dev->data->nb_rx_queues);
2464			return -ENOMEM;
2465		}
2466	}
2467
2468	/* confiugre msix for sleep until rx interrupt */
2469	ixgbe_configure_msix(dev);
2470
2471	/* initialize transmission unit */
2472	ixgbe_dev_tx_init(dev);
2473
2474	/* This can fail when allocating mbufs for descriptor rings */
2475	err = ixgbe_dev_rx_init(dev);
2476	if (err) {
2477		PMD_INIT_LOG(ERR, "Unable to initialize RX hardware");
2478		goto error;
2479	}
2480
2481    mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK |
2482		ETH_VLAN_EXTEND_MASK;
2483	ixgbe_vlan_offload_set(dev, mask);
2484
2485	if (dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_VMDQ_ONLY) {
2486		/* Enable vlan filtering for VMDq */
2487		ixgbe_vmdq_vlan_hw_filter_enable(dev);
2488	}
2489
2490	/* Configure DCB hw */
2491	ixgbe_configure_dcb(dev);
2492
2493	if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_NONE) {
2494		err = ixgbe_fdir_configure(dev);
2495		if (err)
2496			goto error;
2497	}
2498
2499	/* Restore vf rate limit */
2500	if (vfinfo != NULL) {
2501		for (vf = 0; vf < pci_dev->max_vfs; vf++)
2502			for (idx = 0; idx < IXGBE_MAX_QUEUE_NUM_PER_VF; idx++)
2503				if (vfinfo[vf].tx_rate[idx] != 0)
2504					rte_pmd_ixgbe_set_vf_rate_limit(
2505						dev->data->port_id, vf,
2506						vfinfo[vf].tx_rate[idx],
2507						1 << idx);
2508	}
2509
2510	ixgbe_restore_statistics_mapping(dev);
2511
2512	err = ixgbe_dev_rxtx_start(dev);
2513	if (err < 0) {
2514		PMD_INIT_LOG(ERR, "Unable to start rxtx queues");
2515		goto error;
2516	}
2517
2518	/* Skip link setup if loopback mode is enabled for 82599. */
2519	if (hw->mac.type == ixgbe_mac_82599EB &&
2520			dev->data->dev_conf.lpbk_mode == IXGBE_LPBK_82599_TX_RX)
2521		goto skip_link_setup;
2522
2523	if (ixgbe_is_sfp(hw) && hw->phy.multispeed_fiber) {
2524		err = hw->mac.ops.setup_sfp(hw);
2525		if (err)
2526			goto error;
2527	}
2528
2529	if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) {
2530		/* Turn on the copper */
2531		ixgbe_set_phy_power(hw, true);
2532	} else {
2533		/* Turn on the laser */
2534		ixgbe_enable_tx_laser(hw);
2535	}
2536
2537	err = ixgbe_check_link(hw, &speed, &link_up, 0);
2538	if (err)
2539		goto error;
2540	dev->data->dev_link.link_status = link_up;
2541
2542	err = ixgbe_get_link_capabilities(hw, &speed, &negotiate);
2543	if (err)
2544		goto error;
2545
2546	link_speeds = &dev->data->dev_conf.link_speeds;
2547	if (*link_speeds & ~(ETH_LINK_SPEED_100M | ETH_LINK_SPEED_1G |
2548			ETH_LINK_SPEED_10G)) {
2549		PMD_INIT_LOG(ERR, "Invalid link setting");
2550		goto error;
2551	}
2552
2553	speed = 0x0;
2554	if (*link_speeds == ETH_LINK_SPEED_AUTONEG) {
2555		speed = (hw->mac.type != ixgbe_mac_82598EB) ?
2556				IXGBE_LINK_SPEED_82599_AUTONEG :
2557				IXGBE_LINK_SPEED_82598_AUTONEG;
2558	} else {
2559		if (*link_speeds & ETH_LINK_SPEED_10G)
2560			speed |= IXGBE_LINK_SPEED_10GB_FULL;
2561		if (*link_speeds & ETH_LINK_SPEED_1G)
2562			speed |= IXGBE_LINK_SPEED_1GB_FULL;
2563		if (*link_speeds & ETH_LINK_SPEED_100M)
2564			speed |= IXGBE_LINK_SPEED_100_FULL;
2565	}
2566
2567	err = ixgbe_setup_link(hw, speed, link_up);
2568	if (err)
2569		goto error;
2570
2571skip_link_setup:
2572
2573	if (rte_intr_allow_others(intr_handle)) {
2574		/* check if lsc interrupt is enabled */
2575		if (dev->data->dev_conf.intr_conf.lsc != 0)
2576			ixgbe_dev_lsc_interrupt_setup(dev);
2577		ixgbe_dev_macsec_interrupt_setup(dev);
2578	} else {
2579		rte_intr_callback_unregister(intr_handle,
2580					     ixgbe_dev_interrupt_handler, dev);
2581		if (dev->data->dev_conf.intr_conf.lsc != 0)
2582			PMD_INIT_LOG(INFO, "lsc won't enable because of"
2583				     " no intr multiplex");
2584	}
2585
2586	/* check if rxq interrupt is enabled */
2587	if (dev->data->dev_conf.intr_conf.rxq != 0 &&
2588	    rte_intr_dp_is_en(intr_handle))
2589		ixgbe_dev_rxq_interrupt_setup(dev);
2590
2591	/* enable uio/vfio intr/eventfd mapping */
2592	rte_intr_enable(intr_handle);
2593
2594	/* resume enabled intr since hw reset */
2595	ixgbe_enable_intr(dev);
2596	ixgbe_l2_tunnel_conf(dev);
2597	ixgbe_filter_restore(dev);
2598
2599	return 0;
2600
2601error:
2602	PMD_INIT_LOG(ERR, "failure in ixgbe_dev_start(): %d", err);
2603	ixgbe_dev_clear_queues(dev);
2604	return -EIO;
2605}
2606
2607/*
2608 * Stop device: disable rx and tx functions to allow for reconfiguring.
2609 */
2610static void
2611ixgbe_dev_stop(struct rte_eth_dev *dev)
2612{
2613	struct rte_eth_link link;
2614	struct ixgbe_hw *hw =
2615		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2616	struct ixgbe_vf_info *vfinfo =
2617		*IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private);
2618	struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(dev);
2619	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
2620	int vf;
2621
2622	PMD_INIT_FUNC_TRACE();
2623
2624	/* disable interrupts */
2625	ixgbe_disable_intr(hw);
2626
2627	/* reset the NIC */
2628	ixgbe_pf_reset_hw(hw);
2629	hw->adapter_stopped = 0;
2630
2631	/* stop adapter */
2632	ixgbe_stop_adapter(hw);
2633
2634	for (vf = 0; vfinfo != NULL && vf < pci_dev->max_vfs; vf++)
2635		vfinfo[vf].clear_to_send = false;
2636
2637	if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) {
2638		/* Turn off the copper */
2639		ixgbe_set_phy_power(hw, false);
2640	} else {
2641		/* Turn off the laser */
2642		ixgbe_disable_tx_laser(hw);
2643	}
2644
2645	ixgbe_dev_clear_queues(dev);
2646
2647	/* Clear stored conf */
2648	dev->data->scattered_rx = 0;
2649	dev->data->lro = 0;
2650
2651	/* Clear recorded link status */
2652	memset(&link, 0, sizeof(link));
2653	rte_ixgbe_dev_atomic_write_link_status(dev, &link);
2654
2655	if (!rte_intr_allow_others(intr_handle))
2656		/* resume to the default handler */
2657		rte_intr_callback_register(intr_handle,
2658					   ixgbe_dev_interrupt_handler,
2659					   (void *)dev);
2660
2661	/* Clean datapath event and queue/vec mapping */
2662	rte_intr_efd_disable(intr_handle);
2663	if (intr_handle->intr_vec != NULL) {
2664		rte_free(intr_handle->intr_vec);
2665		intr_handle->intr_vec = NULL;
2666	}
2667}
2668
2669/*
2670 * Set device link up: enable tx.
2671 */
2672static int
2673ixgbe_dev_set_link_up(struct rte_eth_dev *dev)
2674{
2675	struct ixgbe_hw *hw =
2676		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2677	if (hw->mac.type == ixgbe_mac_82599EB) {
2678#ifdef RTE_NIC_BYPASS
2679		if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) {
2680			/* Not suported in bypass mode */
2681			PMD_INIT_LOG(ERR, "Set link up is not supported "
2682				     "by device id 0x%x", hw->device_id);
2683			return -ENOTSUP;
2684		}
2685#endif
2686	}
2687
2688	if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) {
2689		/* Turn on the copper */
2690		ixgbe_set_phy_power(hw, true);
2691	} else {
2692		/* Turn on the laser */
2693		ixgbe_enable_tx_laser(hw);
2694	}
2695
2696	return 0;
2697}
2698
2699/*
2700 * Set device link down: disable tx.
2701 */
2702static int
2703ixgbe_dev_set_link_down(struct rte_eth_dev *dev)
2704{
2705	struct ixgbe_hw *hw =
2706		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2707	if (hw->mac.type == ixgbe_mac_82599EB) {
2708#ifdef RTE_NIC_BYPASS
2709		if (hw->device_id == IXGBE_DEV_ID_82599_BYPASS) {
2710			/* Not suported in bypass mode */
2711			PMD_INIT_LOG(ERR, "Set link down is not supported "
2712				     "by device id 0x%x", hw->device_id);
2713			return -ENOTSUP;
2714		}
2715#endif
2716	}
2717
2718	if (hw->mac.ops.get_media_type(hw) == ixgbe_media_type_copper) {
2719		/* Turn off the copper */
2720		ixgbe_set_phy_power(hw, false);
2721	} else {
2722		/* Turn off the laser */
2723		ixgbe_disable_tx_laser(hw);
2724	}
2725
2726	return 0;
2727}
2728
2729/*
2730 * Reest and stop device.
2731 */
2732static void
2733ixgbe_dev_close(struct rte_eth_dev *dev)
2734{
2735	struct ixgbe_hw *hw =
2736		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2737
2738	PMD_INIT_FUNC_TRACE();
2739
2740	ixgbe_pf_reset_hw(hw);
2741
2742	ixgbe_dev_stop(dev);
2743	hw->adapter_stopped = 1;
2744
2745	ixgbe_dev_free_queues(dev);
2746
2747	ixgbe_disable_pcie_master(hw);
2748
2749	/* reprogram the RAR[0] in case user changed it. */
2750	ixgbe_set_rar(hw, 0, hw->mac.addr, 0, IXGBE_RAH_AV);
2751}
2752
2753static void
2754ixgbe_read_stats_registers(struct ixgbe_hw *hw,
2755			   struct ixgbe_hw_stats *hw_stats,
2756			   struct ixgbe_macsec_stats *macsec_stats,
2757			   uint64_t *total_missed_rx, uint64_t *total_qbrc,
2758			   uint64_t *total_qprc, uint64_t *total_qprdc)
2759{
2760	uint32_t bprc, lxon, lxoff, total;
2761	uint32_t delta_gprc = 0;
2762	unsigned i;
2763	/* Workaround for RX byte count not including CRC bytes when CRC
2764	 * strip is enabled. CRC bytes are removed from counters when crc_strip
2765	 * is disabled.
2766	 */
2767	int crc_strip = (IXGBE_READ_REG(hw, IXGBE_HLREG0) &
2768			IXGBE_HLREG0_RXCRCSTRP);
2769
2770	hw_stats->crcerrs += IXGBE_READ_REG(hw, IXGBE_CRCERRS);
2771	hw_stats->illerrc += IXGBE_READ_REG(hw, IXGBE_ILLERRC);
2772	hw_stats->errbc += IXGBE_READ_REG(hw, IXGBE_ERRBC);
2773	hw_stats->mspdc += IXGBE_READ_REG(hw, IXGBE_MSPDC);
2774
2775	for (i = 0; i < 8; i++) {
2776		uint32_t mp = IXGBE_READ_REG(hw, IXGBE_MPC(i));
2777
2778		/* global total per queue */
2779		hw_stats->mpc[i] += mp;
2780		/* Running comprehensive total for stats display */
2781		*total_missed_rx += hw_stats->mpc[i];
2782		if (hw->mac.type == ixgbe_mac_82598EB) {
2783			hw_stats->rnbc[i] +=
2784			    IXGBE_READ_REG(hw, IXGBE_RNBC(i));
2785			hw_stats->pxonrxc[i] +=
2786				IXGBE_READ_REG(hw, IXGBE_PXONRXC(i));
2787			hw_stats->pxoffrxc[i] +=
2788				IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
2789		} else {
2790			hw_stats->pxonrxc[i] +=
2791				IXGBE_READ_REG(hw, IXGBE_PXONRXCNT(i));
2792			hw_stats->pxoffrxc[i] +=
2793				IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i));
2794			hw_stats->pxon2offc[i] +=
2795				IXGBE_READ_REG(hw, IXGBE_PXON2OFFCNT(i));
2796		}
2797		hw_stats->pxontxc[i] +=
2798		    IXGBE_READ_REG(hw, IXGBE_PXONTXC(i));
2799		hw_stats->pxofftxc[i] +=
2800		    IXGBE_READ_REG(hw, IXGBE_PXOFFTXC(i));
2801	}
2802	for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
2803		uint32_t delta_qprc = IXGBE_READ_REG(hw, IXGBE_QPRC(i));
2804		uint32_t delta_qptc = IXGBE_READ_REG(hw, IXGBE_QPTC(i));
2805		uint32_t delta_qprdc = IXGBE_READ_REG(hw, IXGBE_QPRDC(i));
2806
2807		delta_gprc += delta_qprc;
2808
2809		hw_stats->qprc[i] += delta_qprc;
2810		hw_stats->qptc[i] += delta_qptc;
2811
2812		hw_stats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i));
2813		hw_stats->qbrc[i] +=
2814		    ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBRC_H(i)) << 32);
2815		if (crc_strip == 0)
2816			hw_stats->qbrc[i] -= delta_qprc * ETHER_CRC_LEN;
2817
2818		hw_stats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i));
2819		hw_stats->qbtc[i] +=
2820		    ((uint64_t)IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)) << 32);
2821
2822		hw_stats->qprdc[i] += delta_qprdc;
2823		*total_qprdc += hw_stats->qprdc[i];
2824
2825		*total_qprc += hw_stats->qprc[i];
2826		*total_qbrc += hw_stats->qbrc[i];
2827	}
2828	hw_stats->mlfc += IXGBE_READ_REG(hw, IXGBE_MLFC);
2829	hw_stats->mrfc += IXGBE_READ_REG(hw, IXGBE_MRFC);
2830	hw_stats->rlec += IXGBE_READ_REG(hw, IXGBE_RLEC);
2831
2832	/*
2833	 * An errata states that gprc actually counts good + missed packets:
2834	 * Workaround to set gprc to summated queue packet receives
2835	 */
2836	hw_stats->gprc = *total_qprc;
2837
2838	if (hw->mac.type != ixgbe_mac_82598EB) {
2839		hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL);
2840		hw_stats->gorc += ((u64)IXGBE_READ_REG(hw, IXGBE_GORCH) << 32);
2841		hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL);
2842		hw_stats->gotc += ((u64)IXGBE_READ_REG(hw, IXGBE_GOTCH) << 32);
2843		hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORL);
2844		hw_stats->tor += ((u64)IXGBE_READ_REG(hw, IXGBE_TORH) << 32);
2845		hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXCNT);
2846		hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
2847	} else {
2848		hw_stats->lxonrxc += IXGBE_READ_REG(hw, IXGBE_LXONRXC);
2849		hw_stats->lxoffrxc += IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
2850		/* 82598 only has a counter in the high register */
2851		hw_stats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCH);
2852		hw_stats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCH);
2853		hw_stats->tor += IXGBE_READ_REG(hw, IXGBE_TORH);
2854	}
2855	uint64_t old_tpr = hw_stats->tpr;
2856
2857	hw_stats->tpr += IXGBE_READ_REG(hw, IXGBE_TPR);
2858	hw_stats->tpt += IXGBE_READ_REG(hw, IXGBE_TPT);
2859
2860	if (crc_strip == 0)
2861		hw_stats->gorc -= delta_gprc * ETHER_CRC_LEN;
2862
2863	uint64_t delta_gptc = IXGBE_READ_REG(hw, IXGBE_GPTC);
2864	hw_stats->gptc += delta_gptc;
2865	hw_stats->gotc -= delta_gptc * ETHER_CRC_LEN;
2866	hw_stats->tor -= (hw_stats->tpr - old_tpr) * ETHER_CRC_LEN;
2867
2868	/*
2869	 * Workaround: mprc hardware is incorrectly counting
2870	 * broadcasts, so for now we subtract those.
2871	 */
2872	bprc = IXGBE_READ_REG(hw, IXGBE_BPRC);
2873	hw_stats->bprc += bprc;
2874	hw_stats->mprc += IXGBE_READ_REG(hw, IXGBE_MPRC);
2875	if (hw->mac.type == ixgbe_mac_82598EB)
2876		hw_stats->mprc -= bprc;
2877
2878	hw_stats->prc64 += IXGBE_READ_REG(hw, IXGBE_PRC64);
2879	hw_stats->prc127 += IXGBE_READ_REG(hw, IXGBE_PRC127);
2880	hw_stats->prc255 += IXGBE_READ_REG(hw, IXGBE_PRC255);
2881	hw_stats->prc511 += IXGBE_READ_REG(hw, IXGBE_PRC511);
2882	hw_stats->prc1023 += IXGBE_READ_REG(hw, IXGBE_PRC1023);
2883	hw_stats->prc1522 += IXGBE_READ_REG(hw, IXGBE_PRC1522);
2884
2885	lxon = IXGBE_READ_REG(hw, IXGBE_LXONTXC);
2886	hw_stats->lxontxc += lxon;
2887	lxoff = IXGBE_READ_REG(hw, IXGBE_LXOFFTXC);
2888	hw_stats->lxofftxc += lxoff;
2889	total = lxon + lxoff;
2890
2891	hw_stats->mptc += IXGBE_READ_REG(hw, IXGBE_MPTC);
2892	hw_stats->ptc64 += IXGBE_READ_REG(hw, IXGBE_PTC64);
2893	hw_stats->gptc -= total;
2894	hw_stats->mptc -= total;
2895	hw_stats->ptc64 -= total;
2896	hw_stats->gotc -= total * ETHER_MIN_LEN;
2897
2898	hw_stats->ruc += IXGBE_READ_REG(hw, IXGBE_RUC);
2899	hw_stats->rfc += IXGBE_READ_REG(hw, IXGBE_RFC);
2900	hw_stats->roc += IXGBE_READ_REG(hw, IXGBE_ROC);
2901	hw_stats->rjc += IXGBE_READ_REG(hw, IXGBE_RJC);
2902	hw_stats->mngprc += IXGBE_READ_REG(hw, IXGBE_MNGPRC);
2903	hw_stats->mngpdc += IXGBE_READ_REG(hw, IXGBE_MNGPDC);
2904	hw_stats->mngptc += IXGBE_READ_REG(hw, IXGBE_MNGPTC);
2905	hw_stats->ptc127 += IXGBE_READ_REG(hw, IXGBE_PTC127);
2906	hw_stats->ptc255 += IXGBE_READ_REG(hw, IXGBE_PTC255);
2907	hw_stats->ptc511 += IXGBE_READ_REG(hw, IXGBE_PTC511);
2908	hw_stats->ptc1023 += IXGBE_READ_REG(hw, IXGBE_PTC1023);
2909	hw_stats->ptc1522 += IXGBE_READ_REG(hw, IXGBE_PTC1522);
2910	hw_stats->bptc += IXGBE_READ_REG(hw, IXGBE_BPTC);
2911	hw_stats->xec += IXGBE_READ_REG(hw, IXGBE_XEC);
2912	hw_stats->fccrc += IXGBE_READ_REG(hw, IXGBE_FCCRC);
2913	hw_stats->fclast += IXGBE_READ_REG(hw, IXGBE_FCLAST);
2914	/* Only read FCOE on 82599 */
2915	if (hw->mac.type != ixgbe_mac_82598EB) {
2916		hw_stats->fcoerpdc += IXGBE_READ_REG(hw, IXGBE_FCOERPDC);
2917		hw_stats->fcoeprc += IXGBE_READ_REG(hw, IXGBE_FCOEPRC);
2918		hw_stats->fcoeptc += IXGBE_READ_REG(hw, IXGBE_FCOEPTC);
2919		hw_stats->fcoedwrc += IXGBE_READ_REG(hw, IXGBE_FCOEDWRC);
2920		hw_stats->fcoedwtc += IXGBE_READ_REG(hw, IXGBE_FCOEDWTC);
2921	}
2922
2923	/* Flow Director Stats registers */
2924	hw_stats->fdirmatch += IXGBE_READ_REG(hw, IXGBE_FDIRMATCH);
2925	hw_stats->fdirmiss += IXGBE_READ_REG(hw, IXGBE_FDIRMISS);
2926
2927	/* MACsec Stats registers */
2928	macsec_stats->out_pkts_untagged += IXGBE_READ_REG(hw, IXGBE_LSECTXUT);
2929	macsec_stats->out_pkts_encrypted +=
2930		IXGBE_READ_REG(hw, IXGBE_LSECTXPKTE);
2931	macsec_stats->out_pkts_protected +=
2932		IXGBE_READ_REG(hw, IXGBE_LSECTXPKTP);
2933	macsec_stats->out_octets_encrypted +=
2934		IXGBE_READ_REG(hw, IXGBE_LSECTXOCTE);
2935	macsec_stats->out_octets_protected +=
2936		IXGBE_READ_REG(hw, IXGBE_LSECTXOCTP);
2937	macsec_stats->in_pkts_untagged += IXGBE_READ_REG(hw, IXGBE_LSECRXUT);
2938	macsec_stats->in_pkts_badtag += IXGBE_READ_REG(hw, IXGBE_LSECRXBAD);
2939	macsec_stats->in_pkts_nosci += IXGBE_READ_REG(hw, IXGBE_LSECRXNOSCI);
2940	macsec_stats->in_pkts_unknownsci +=
2941		IXGBE_READ_REG(hw, IXGBE_LSECRXUNSCI);
2942	macsec_stats->in_octets_decrypted +=
2943		IXGBE_READ_REG(hw, IXGBE_LSECRXOCTD);
2944	macsec_stats->in_octets_validated +=
2945		IXGBE_READ_REG(hw, IXGBE_LSECRXOCTV);
2946	macsec_stats->in_pkts_unchecked += IXGBE_READ_REG(hw, IXGBE_LSECRXUNCH);
2947	macsec_stats->in_pkts_delayed += IXGBE_READ_REG(hw, IXGBE_LSECRXDELAY);
2948	macsec_stats->in_pkts_late += IXGBE_READ_REG(hw, IXGBE_LSECRXLATE);
2949	for (i = 0; i < 2; i++) {
2950		macsec_stats->in_pkts_ok +=
2951			IXGBE_READ_REG(hw, IXGBE_LSECRXOK(i));
2952		macsec_stats->in_pkts_invalid +=
2953			IXGBE_READ_REG(hw, IXGBE_LSECRXINV(i));
2954		macsec_stats->in_pkts_notvalid +=
2955			IXGBE_READ_REG(hw, IXGBE_LSECRXNV(i));
2956	}
2957	macsec_stats->in_pkts_unusedsa += IXGBE_READ_REG(hw, IXGBE_LSECRXUNSA);
2958	macsec_stats->in_pkts_notusingsa +=
2959		IXGBE_READ_REG(hw, IXGBE_LSECRXNUSA);
2960}
2961
2962/*
2963 * This function is based on ixgbe_update_stats_counters() in ixgbe/ixgbe.c
2964 */
2965static void
2966ixgbe_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
2967{
2968	struct ixgbe_hw *hw =
2969			IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
2970	struct ixgbe_hw_stats *hw_stats =
2971			IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
2972	struct ixgbe_macsec_stats *macsec_stats =
2973			IXGBE_DEV_PRIVATE_TO_MACSEC_STATS(
2974				dev->data->dev_private);
2975	uint64_t total_missed_rx, total_qbrc, total_qprc, total_qprdc;
2976	unsigned i;
2977
2978	total_missed_rx = 0;
2979	total_qbrc = 0;
2980	total_qprc = 0;
2981	total_qprdc = 0;
2982
2983	ixgbe_read_stats_registers(hw, hw_stats, macsec_stats, &total_missed_rx,
2984			&total_qbrc, &total_qprc, &total_qprdc);
2985
2986	if (stats == NULL)
2987		return;
2988
2989	/* Fill out the rte_eth_stats statistics structure */
2990	stats->ipackets = total_qprc;
2991	stats->ibytes = total_qbrc;
2992	stats->opackets = hw_stats->gptc;
2993	stats->obytes = hw_stats->gotc;
2994
2995	for (i = 0; i < IXGBE_QUEUE_STAT_COUNTERS; i++) {
2996		stats->q_ipackets[i] = hw_stats->qprc[i];
2997		stats->q_opackets[i] = hw_stats->qptc[i];
2998		stats->q_ibytes[i] = hw_stats->qbrc[i];
2999		stats->q_obytes[i] = hw_stats->qbtc[i];
3000		stats->q_errors[i] = hw_stats->qprdc[i];
3001	}
3002
3003	/* Rx Errors */
3004	stats->imissed  = total_missed_rx;
3005	stats->ierrors  = hw_stats->crcerrs +
3006			  hw_stats->mspdc +
3007			  hw_stats->rlec +
3008			  hw_stats->ruc +
3009			  hw_stats->roc +
3010			  hw_stats->illerrc +
3011			  hw_stats->errbc +
3012			  hw_stats->rfc +
3013			  hw_stats->fccrc +
3014			  hw_stats->fclast;
3015
3016	/* Tx Errors */
3017	stats->oerrors  = 0;
3018}
3019
3020static void
3021ixgbe_dev_stats_reset(struct rte_eth_dev *dev)
3022{
3023	struct ixgbe_hw_stats *stats =
3024			IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
3025
3026	/* HW registers are cleared on read */
3027	ixgbe_dev_stats_get(dev, NULL);
3028
3029	/* Reset software totals */
3030	memset(stats, 0, sizeof(*stats));
3031}
3032
3033/* This function calculates the number of xstats based on the current config */
3034static unsigned
3035ixgbe_xstats_calc_num(void) {
3036	return IXGBE_NB_HW_STATS + IXGBE_NB_MACSEC_STATS +
3037		(IXGBE_NB_RXQ_PRIO_STATS * IXGBE_NB_RXQ_PRIO_VALUES) +
3038		(IXGBE_NB_TXQ_PRIO_STATS * IXGBE_NB_TXQ_PRIO_VALUES);
3039}
3040
3041static int ixgbe_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
3042	struct rte_eth_xstat_name *xstats_names, __rte_unused unsigned limit)
3043{
3044	const unsigned cnt_stats = ixgbe_xstats_calc_num();
3045	unsigned stat, i, count;
3046
3047	if (xstats_names != NULL) {
3048		count = 0;
3049
3050		/* Note: limit >= cnt_stats checked upstream
3051		 * in rte_eth_xstats_names()
3052		 */
3053
3054		/* Extended stats from ixgbe_hw_stats */
3055		for (i = 0; i < IXGBE_NB_HW_STATS; i++) {
3056			snprintf(xstats_names[count].name,
3057				sizeof(xstats_names[count].name),
3058				"%s",
3059				rte_ixgbe_stats_strings[i].name);
3060			count++;
3061		}
3062
3063		/* MACsec Stats */
3064		for (i = 0; i < IXGBE_NB_MACSEC_STATS; i++) {
3065			snprintf(xstats_names[count].name,
3066				sizeof(xstats_names[count].name),
3067				"%s",
3068				rte_ixgbe_macsec_strings[i].name);
3069			count++;
3070		}
3071
3072		/* RX Priority Stats */
3073		for (stat = 0; stat < IXGBE_NB_RXQ_PRIO_STATS; stat++) {
3074			for (i = 0; i < IXGBE_NB_RXQ_PRIO_VALUES; i++) {
3075				snprintf(xstats_names[count].name,
3076					sizeof(xstats_names[count].name),
3077					"rx_priority%u_%s", i,
3078					rte_ixgbe_rxq_strings[stat].name);
3079				count++;
3080			}
3081		}
3082
3083		/* TX Priority Stats */
3084		for (stat = 0; stat < IXGBE_NB_TXQ_PRIO_STATS; stat++) {
3085			for (i = 0; i < IXGBE_NB_TXQ_PRIO_VALUES; i++) {
3086				snprintf(xstats_names[count].name,
3087					sizeof(xstats_names[count].name),
3088					"tx_priority%u_%s", i,
3089					rte_ixgbe_txq_strings[stat].name);
3090				count++;
3091			}
3092		}
3093	}
3094	return cnt_stats;
3095}
3096
3097static int ixgbevf_dev_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
3098	struct rte_eth_xstat_name *xstats_names, unsigned limit)
3099{
3100	unsigned i;
3101
3102	if (limit < IXGBEVF_NB_XSTATS && xstats_names != NULL)
3103		return -ENOMEM;
3104
3105	if (xstats_names != NULL)
3106		for (i = 0; i < IXGBEVF_NB_XSTATS; i++)
3107			snprintf(xstats_names[i].name,
3108				sizeof(xstats_names[i].name),
3109				"%s", rte_ixgbevf_stats_strings[i].name);
3110	return IXGBEVF_NB_XSTATS;
3111}
3112
3113static int
3114ixgbe_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
3115					 unsigned n)
3116{
3117	struct ixgbe_hw *hw =
3118			IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3119	struct ixgbe_hw_stats *hw_stats =
3120			IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
3121	struct ixgbe_macsec_stats *macsec_stats =
3122			IXGBE_DEV_PRIVATE_TO_MACSEC_STATS(
3123				dev->data->dev_private);
3124	uint64_t total_missed_rx, total_qbrc, total_qprc, total_qprdc;
3125	unsigned i, stat, count = 0;
3126
3127	count = ixgbe_xstats_calc_num();
3128
3129	if (n < count)
3130		return count;
3131
3132	total_missed_rx = 0;
3133	total_qbrc = 0;
3134	total_qprc = 0;
3135	total_qprdc = 0;
3136
3137	ixgbe_read_stats_registers(hw, hw_stats, macsec_stats, &total_missed_rx,
3138			&total_qbrc, &total_qprc, &total_qprdc);
3139
3140	/* If this is a reset xstats is NULL, and we have cleared the
3141	 * registers by reading them.
3142	 */
3143	if (!xstats)
3144		return 0;
3145
3146	/* Extended stats from ixgbe_hw_stats */
3147	count = 0;
3148	for (i = 0; i < IXGBE_NB_HW_STATS; i++) {
3149		xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
3150				rte_ixgbe_stats_strings[i].offset);
3151		xstats[count].id = count;
3152		count++;
3153	}
3154
3155	/* MACsec Stats */
3156	for (i = 0; i < IXGBE_NB_MACSEC_STATS; i++) {
3157		xstats[count].value = *(uint64_t *)(((char *)macsec_stats) +
3158				rte_ixgbe_macsec_strings[i].offset);
3159		xstats[count].id = count;
3160		count++;
3161	}
3162
3163	/* RX Priority Stats */
3164	for (stat = 0; stat < IXGBE_NB_RXQ_PRIO_STATS; stat++) {
3165		for (i = 0; i < IXGBE_NB_RXQ_PRIO_VALUES; i++) {
3166			xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
3167					rte_ixgbe_rxq_strings[stat].offset +
3168					(sizeof(uint64_t) * i));
3169			xstats[count].id = count;
3170			count++;
3171		}
3172	}
3173
3174	/* TX Priority Stats */
3175	for (stat = 0; stat < IXGBE_NB_TXQ_PRIO_STATS; stat++) {
3176		for (i = 0; i < IXGBE_NB_TXQ_PRIO_VALUES; i++) {
3177			xstats[count].value = *(uint64_t *)(((char *)hw_stats) +
3178					rte_ixgbe_txq_strings[stat].offset +
3179					(sizeof(uint64_t) * i));
3180			xstats[count].id = count;
3181			count++;
3182		}
3183	}
3184	return count;
3185}
3186
3187static void
3188ixgbe_dev_xstats_reset(struct rte_eth_dev *dev)
3189{
3190	struct ixgbe_hw_stats *stats =
3191			IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
3192	struct ixgbe_macsec_stats *macsec_stats =
3193			IXGBE_DEV_PRIVATE_TO_MACSEC_STATS(
3194				dev->data->dev_private);
3195
3196	unsigned count = ixgbe_xstats_calc_num();
3197
3198	/* HW registers are cleared on read */
3199	ixgbe_dev_xstats_get(dev, NULL, count);
3200
3201	/* Reset software totals */
3202	memset(stats, 0, sizeof(*stats));
3203	memset(macsec_stats, 0, sizeof(*macsec_stats));
3204}
3205
3206static void
3207ixgbevf_update_stats(struct rte_eth_dev *dev)
3208{
3209	struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3210	struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats *)
3211			  IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
3212
3213	/* Good Rx packet, include VF loopback */
3214	UPDATE_VF_STAT(IXGBE_VFGPRC,
3215	    hw_stats->last_vfgprc, hw_stats->vfgprc);
3216
3217	/* Good Rx octets, include VF loopback */
3218	UPDATE_VF_STAT_36BIT(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
3219	    hw_stats->last_vfgorc, hw_stats->vfgorc);
3220
3221	/* Good Tx packet, include VF loopback */
3222	UPDATE_VF_STAT(IXGBE_VFGPTC,
3223	    hw_stats->last_vfgptc, hw_stats->vfgptc);
3224
3225	/* Good Tx octets, include VF loopback */
3226	UPDATE_VF_STAT_36BIT(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
3227	    hw_stats->last_vfgotc, hw_stats->vfgotc);
3228
3229	/* Rx Multicst Packet */
3230	UPDATE_VF_STAT(IXGBE_VFMPRC,
3231	    hw_stats->last_vfmprc, hw_stats->vfmprc);
3232}
3233
3234static int
3235ixgbevf_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
3236		       unsigned n)
3237{
3238	struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats *)
3239			IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
3240	unsigned i;
3241
3242	if (n < IXGBEVF_NB_XSTATS)
3243		return IXGBEVF_NB_XSTATS;
3244
3245	ixgbevf_update_stats(dev);
3246
3247	if (!xstats)
3248		return 0;
3249
3250	/* Extended stats */
3251	for (i = 0; i < IXGBEVF_NB_XSTATS; i++) {
3252		xstats[i].value = *(uint64_t *)(((char *)hw_stats) +
3253			rte_ixgbevf_stats_strings[i].offset);
3254	}
3255
3256	return IXGBEVF_NB_XSTATS;
3257}
3258
3259static void
3260ixgbevf_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
3261{
3262	struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats *)
3263			  IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
3264
3265	ixgbevf_update_stats(dev);
3266
3267	if (stats == NULL)
3268		return;
3269
3270	stats->ipackets = hw_stats->vfgprc;
3271	stats->ibytes = hw_stats->vfgorc;
3272	stats->opackets = hw_stats->vfgptc;
3273	stats->obytes = hw_stats->vfgotc;
3274}
3275
3276static void
3277ixgbevf_dev_stats_reset(struct rte_eth_dev *dev)
3278{
3279	struct ixgbevf_hw_stats *hw_stats = (struct ixgbevf_hw_stats *)
3280			IXGBE_DEV_PRIVATE_TO_STATS(dev->data->dev_private);
3281
3282	/* Sync HW register to the last stats */
3283	ixgbevf_dev_stats_get(dev, NULL);
3284
3285	/* reset HW current stats*/
3286	hw_stats->vfgprc = 0;
3287	hw_stats->vfgorc = 0;
3288	hw_stats->vfgptc = 0;
3289	hw_stats->vfgotc = 0;
3290}
3291
3292static int
3293ixgbe_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size)
3294{
3295	struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3296	u16 eeprom_verh, eeprom_verl;
3297	u32 etrack_id;
3298	int ret;
3299
3300	ixgbe_read_eeprom(hw, 0x2e, &eeprom_verh);
3301	ixgbe_read_eeprom(hw, 0x2d, &eeprom_verl);
3302
3303	etrack_id = (eeprom_verh << 16) | eeprom_verl;
3304	ret = snprintf(fw_version, fw_size, "0x%08x", etrack_id);
3305
3306	ret += 1; /* add the size of '\0' */
3307	if (fw_size < (u32)ret)
3308		return ret;
3309	else
3310		return 0;
3311}
3312
3313static void
3314ixgbe_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
3315{
3316	struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(dev);
3317	struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3318	struct rte_eth_conf *dev_conf = &dev->data->dev_conf;
3319
3320	dev_info->pci_dev = pci_dev;
3321	dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
3322	dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues;
3323	if (RTE_ETH_DEV_SRIOV(dev).active == 0) {
3324		/*
3325		 * When DCB/VT is off, maximum number of queues changes,
3326		 * except for 82598EB, which remains constant.
3327		 */
3328		if (dev_conf->txmode.mq_mode == ETH_MQ_TX_NONE &&
3329				hw->mac.type != ixgbe_mac_82598EB)
3330			dev_info->max_tx_queues = IXGBE_NONE_MODE_TX_NB_QUEUES;
3331	}
3332	dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL register */
3333	dev_info->max_rx_pktlen = 15872; /* includes CRC, cf MAXFRS register */
3334	dev_info->max_mac_addrs = hw->mac.num_rar_entries;
3335	dev_info->max_hash_mac_addrs = IXGBE_VMDQ_NUM_UC_MAC;
3336	dev_info->max_vfs = pci_dev->max_vfs;
3337	if (hw->mac.type == ixgbe_mac_82598EB)
3338		dev_info->max_vmdq_pools = ETH_16_POOLS;
3339	else
3340		dev_info->max_vmdq_pools = ETH_64_POOLS;
3341	dev_info->vmdq_queue_num = dev_info->max_rx_queues;
3342	dev_info->rx_offload_capa =
3343		DEV_RX_OFFLOAD_VLAN_STRIP |
3344		DEV_RX_OFFLOAD_IPV4_CKSUM |
3345		DEV_RX_OFFLOAD_UDP_CKSUM  |
3346		DEV_RX_OFFLOAD_TCP_CKSUM;
3347
3348	/*
3349	 * RSC is only supported by 82599 and x540 PF devices in a non-SR-IOV
3350	 * mode.
3351	 */
3352	if ((hw->mac.type == ixgbe_mac_82599EB ||
3353	     hw->mac.type == ixgbe_mac_X540) &&
3354	    !RTE_ETH_DEV_SRIOV(dev).active)
3355		dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TCP_LRO;
3356
3357	if (hw->mac.type == ixgbe_mac_82599EB ||
3358	    hw->mac.type == ixgbe_mac_X540)
3359		dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_MACSEC_STRIP;
3360
3361	if (hw->mac.type == ixgbe_mac_X550 ||
3362	    hw->mac.type == ixgbe_mac_X550EM_x ||
3363	    hw->mac.type == ixgbe_mac_X550EM_a)
3364		dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM;
3365
3366	dev_info->tx_offload_capa =
3367		DEV_TX_OFFLOAD_VLAN_INSERT |
3368		DEV_TX_OFFLOAD_IPV4_CKSUM  |
3369		DEV_TX_OFFLOAD_UDP_CKSUM   |
3370		DEV_TX_OFFLOAD_TCP_CKSUM   |
3371		DEV_TX_OFFLOAD_SCTP_CKSUM  |
3372		DEV_TX_OFFLOAD_TCP_TSO;
3373
3374	if (hw->mac.type == ixgbe_mac_82599EB ||
3375	    hw->mac.type == ixgbe_mac_X540)
3376		dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_MACSEC_INSERT;
3377
3378	if (hw->mac.type == ixgbe_mac_X550 ||
3379	    hw->mac.type == ixgbe_mac_X550EM_x ||
3380	    hw->mac.type == ixgbe_mac_X550EM_a)
3381		dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM;
3382
3383	dev_info->default_rxconf = (struct rte_eth_rxconf) {
3384		.rx_thresh = {
3385			.pthresh = IXGBE_DEFAULT_RX_PTHRESH,
3386			.hthresh = IXGBE_DEFAULT_RX_HTHRESH,
3387			.wthresh = IXGBE_DEFAULT_RX_WTHRESH,
3388		},
3389		.rx_free_thresh = IXGBE_DEFAULT_RX_FREE_THRESH,
3390		.rx_drop_en = 0,
3391	};
3392
3393	dev_info->default_txconf = (struct rte_eth_txconf) {
3394		.tx_thresh = {
3395			.pthresh = IXGBE_DEFAULT_TX_PTHRESH,
3396			.hthresh = IXGBE_DEFAULT_TX_HTHRESH,
3397			.wthresh = IXGBE_DEFAULT_TX_WTHRESH,
3398		},
3399		.tx_free_thresh = IXGBE_DEFAULT_TX_FREE_THRESH,
3400		.tx_rs_thresh = IXGBE_DEFAULT_TX_RSBIT_THRESH,
3401		.txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
3402				ETH_TXQ_FLAGS_NOOFFLOADS,
3403	};
3404
3405	dev_info->rx_desc_lim = rx_desc_lim;
3406	dev_info->tx_desc_lim = tx_desc_lim;
3407
3408	dev_info->hash_key_size = IXGBE_HKEY_MAX_INDEX * sizeof(uint32_t);
3409	dev_info->reta_size = ixgbe_reta_size_get(hw->mac.type);
3410	dev_info->flow_type_rss_offloads = IXGBE_RSS_OFFLOAD_ALL;
3411
3412	dev_info->speed_capa = ETH_LINK_SPEED_1G | ETH_LINK_SPEED_10G;
3413	if (hw->mac.type == ixgbe_mac_X540 ||
3414	    hw->mac.type == ixgbe_mac_X540_vf ||
3415	    hw->mac.type == ixgbe_mac_X550 ||
3416	    hw->mac.type == ixgbe_mac_X550_vf) {
3417		dev_info->speed_capa |= ETH_LINK_SPEED_100M;
3418	}
3419}
3420
3421static const uint32_t *
3422ixgbe_dev_supported_ptypes_get(struct rte_eth_dev *dev)
3423{
3424	static const uint32_t ptypes[] = {
3425		/* For non-vec functions,
3426		 * refers to ixgbe_rxd_pkt_info_to_pkt_type();
3427		 * for vec functions,
3428		 * refers to _recv_raw_pkts_vec().
3429		 */
3430		RTE_PTYPE_L2_ETHER,
3431		RTE_PTYPE_L3_IPV4,
3432		RTE_PTYPE_L3_IPV4_EXT,
3433		RTE_PTYPE_L3_IPV6,
3434		RTE_PTYPE_L3_IPV6_EXT,
3435		RTE_PTYPE_L4_SCTP,
3436		RTE_PTYPE_L4_TCP,
3437		RTE_PTYPE_L4_UDP,
3438		RTE_PTYPE_TUNNEL_IP,
3439		RTE_PTYPE_INNER_L3_IPV6,
3440		RTE_PTYPE_INNER_L3_IPV6_EXT,
3441		RTE_PTYPE_INNER_L4_TCP,
3442		RTE_PTYPE_INNER_L4_UDP,
3443		RTE_PTYPE_UNKNOWN
3444	};
3445
3446	if (dev->rx_pkt_burst == ixgbe_recv_pkts ||
3447	    dev->rx_pkt_burst == ixgbe_recv_pkts_lro_single_alloc ||
3448	    dev->rx_pkt_burst == ixgbe_recv_pkts_lro_bulk_alloc ||
3449	    dev->rx_pkt_burst == ixgbe_recv_pkts_bulk_alloc)
3450		return ptypes;
3451	return NULL;
3452}
3453
3454static void
3455ixgbevf_dev_info_get(struct rte_eth_dev *dev,
3456		     struct rte_eth_dev_info *dev_info)
3457{
3458	struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(dev);
3459	struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3460
3461	dev_info->pci_dev = pci_dev;
3462	dev_info->max_rx_queues = (uint16_t)hw->mac.max_rx_queues;
3463	dev_info->max_tx_queues = (uint16_t)hw->mac.max_tx_queues;
3464	dev_info->min_rx_bufsize = 1024; /* cf BSIZEPACKET in SRRCTL reg */
3465	dev_info->max_rx_pktlen = 9728; /* includes CRC, cf MAXFRS reg */
3466	dev_info->max_mac_addrs = hw->mac.num_rar_entries;
3467	dev_info->max_hash_mac_addrs = IXGBE_VMDQ_NUM_UC_MAC;
3468	dev_info->max_vfs = pci_dev->max_vfs;
3469	if (hw->mac.type == ixgbe_mac_82598EB)
3470		dev_info->max_vmdq_pools = ETH_16_POOLS;
3471	else
3472		dev_info->max_vmdq_pools = ETH_64_POOLS;
3473	dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
3474				DEV_RX_OFFLOAD_IPV4_CKSUM |
3475				DEV_RX_OFFLOAD_UDP_CKSUM  |
3476				DEV_RX_OFFLOAD_TCP_CKSUM;
3477	dev_info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
3478				DEV_TX_OFFLOAD_IPV4_CKSUM  |
3479				DEV_TX_OFFLOAD_UDP_CKSUM   |
3480				DEV_TX_OFFLOAD_TCP_CKSUM   |
3481				DEV_TX_OFFLOAD_SCTP_CKSUM  |
3482				DEV_TX_OFFLOAD_TCP_TSO;
3483
3484	dev_info->default_rxconf = (struct rte_eth_rxconf) {
3485		.rx_thresh = {
3486			.pthresh = IXGBE_DEFAULT_RX_PTHRESH,
3487			.hthresh = IXGBE_DEFAULT_RX_HTHRESH,
3488			.wthresh = IXGBE_DEFAULT_RX_WTHRESH,
3489		},
3490		.rx_free_thresh = IXGBE_DEFAULT_RX_FREE_THRESH,
3491		.rx_drop_en = 0,
3492	};
3493
3494	dev_info->default_txconf = (struct rte_eth_txconf) {
3495		.tx_thresh = {
3496			.pthresh = IXGBE_DEFAULT_TX_PTHRESH,
3497			.hthresh = IXGBE_DEFAULT_TX_HTHRESH,
3498			.wthresh = IXGBE_DEFAULT_TX_WTHRESH,
3499		},
3500		.tx_free_thresh = IXGBE_DEFAULT_TX_FREE_THRESH,
3501		.tx_rs_thresh = IXGBE_DEFAULT_TX_RSBIT_THRESH,
3502		.txq_flags = ETH_TXQ_FLAGS_NOMULTSEGS |
3503				ETH_TXQ_FLAGS_NOOFFLOADS,
3504	};
3505
3506	dev_info->rx_desc_lim = rx_desc_lim;
3507	dev_info->tx_desc_lim = tx_desc_lim;
3508}
3509
3510/* return 0 means link status changed, -1 means not changed */
3511static int
3512ixgbe_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
3513{
3514	struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3515	struct rte_eth_link link, old;
3516	ixgbe_link_speed link_speed = IXGBE_LINK_SPEED_UNKNOWN;
3517	int link_up;
3518	int diag;
3519
3520	link.link_status = ETH_LINK_DOWN;
3521	link.link_speed = 0;
3522	link.link_duplex = ETH_LINK_HALF_DUPLEX;
3523	memset(&old, 0, sizeof(old));
3524	rte_ixgbe_dev_atomic_read_link_status(dev, &old);
3525
3526	hw->mac.get_link_status = true;
3527
3528	/* check if it needs to wait to complete, if lsc interrupt is enabled */
3529	if (wait_to_complete == 0 || dev->data->dev_conf.intr_conf.lsc != 0)
3530		diag = ixgbe_check_link(hw, &link_speed, &link_up, 0);
3531	else
3532		diag = ixgbe_check_link(hw, &link_speed, &link_up, 1);
3533
3534	if (diag != 0) {
3535		link.link_speed = ETH_SPEED_NUM_100M;
3536		link.link_duplex = ETH_LINK_FULL_DUPLEX;
3537		rte_ixgbe_dev_atomic_write_link_status(dev, &link);
3538		if (link.link_status == old.link_status)
3539			return -1;
3540		return 0;
3541	}
3542
3543	if (link_up == 0) {
3544		rte_ixgbe_dev_atomic_write_link_status(dev, &link);
3545		if (link.link_status == old.link_status)
3546			return -1;
3547		return 0;
3548	}
3549	link.link_status = ETH_LINK_UP;
3550	link.link_duplex = ETH_LINK_FULL_DUPLEX;
3551
3552	switch (link_speed) {
3553	default:
3554	case IXGBE_LINK_SPEED_UNKNOWN:
3555		link.link_duplex = ETH_LINK_FULL_DUPLEX;
3556		link.link_speed = ETH_SPEED_NUM_100M;
3557		break;
3558
3559	case IXGBE_LINK_SPEED_100_FULL:
3560		link.link_speed = ETH_SPEED_NUM_100M;
3561		break;
3562
3563	case IXGBE_LINK_SPEED_1GB_FULL:
3564		link.link_speed = ETH_SPEED_NUM_1G;
3565		break;
3566
3567	case IXGBE_LINK_SPEED_10GB_FULL:
3568		link.link_speed = ETH_SPEED_NUM_10G;
3569		break;
3570	}
3571	rte_ixgbe_dev_atomic_write_link_status(dev, &link);
3572
3573	if (link.link_status == old.link_status)
3574		return -1;
3575
3576	return 0;
3577}
3578
3579static void
3580ixgbe_dev_promiscuous_enable(struct rte_eth_dev *dev)
3581{
3582	struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3583	uint32_t fctrl;
3584
3585	fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
3586	fctrl |= (IXGBE_FCTRL_UPE | IXGBE_FCTRL_MPE);
3587	IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
3588}
3589
3590static void
3591ixgbe_dev_promiscuous_disable(struct rte_eth_dev *dev)
3592{
3593	struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3594	uint32_t fctrl;
3595
3596	fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
3597	fctrl &= (~IXGBE_FCTRL_UPE);
3598	if (dev->data->all_multicast == 1)
3599		fctrl |= IXGBE_FCTRL_MPE;
3600	else
3601		fctrl &= (~IXGBE_FCTRL_MPE);
3602	IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
3603}
3604
3605static void
3606ixgbe_dev_allmulticast_enable(struct rte_eth_dev *dev)
3607{
3608	struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3609	uint32_t fctrl;
3610
3611	fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
3612	fctrl |= IXGBE_FCTRL_MPE;
3613	IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
3614}
3615
3616static void
3617ixgbe_dev_allmulticast_disable(struct rte_eth_dev *dev)
3618{
3619	struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3620	uint32_t fctrl;
3621
3622	if (dev->data->promiscuous == 1)
3623		return; /* must remain in all_multicast mode */
3624
3625	fctrl = IXGBE_READ_REG(hw, IXGBE_FCTRL);
3626	fctrl &= (~IXGBE_FCTRL_MPE);
3627	IXGBE_WRITE_REG(hw, IXGBE_FCTRL, fctrl);
3628}
3629
3630/**
3631 * It clears the interrupt causes and enables the interrupt.
3632 * It will be called once only during nic initialized.
3633 *
3634 * @param dev
3635 *  Pointer to struct rte_eth_dev.
3636 *
3637 * @return
3638 *  - On success, zero.
3639 *  - On failure, a negative value.
3640 */
3641static int
3642ixgbe_dev_lsc_interrupt_setup(struct rte_eth_dev *dev)
3643{
3644	struct ixgbe_interrupt *intr =
3645		IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
3646
3647	ixgbe_dev_link_status_print(dev);
3648	intr->mask |= IXGBE_EICR_LSC;
3649
3650	return 0;
3651}
3652
3653/**
3654 * It clears the interrupt causes and enables the interrupt.
3655 * It will be called once only during nic initialized.
3656 *
3657 * @param dev
3658 *  Pointer to struct rte_eth_dev.
3659 *
3660 * @return
3661 *  - On success, zero.
3662 *  - On failure, a negative value.
3663 */
3664static int
3665ixgbe_dev_rxq_interrupt_setup(struct rte_eth_dev *dev)
3666{
3667	struct ixgbe_interrupt *intr =
3668		IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
3669
3670	intr->mask |= IXGBE_EICR_RTX_QUEUE;
3671
3672	return 0;
3673}
3674
3675/**
3676 * It clears the interrupt causes and enables the interrupt.
3677 * It will be called once only during nic initialized.
3678 *
3679 * @param dev
3680 *  Pointer to struct rte_eth_dev.
3681 *
3682 * @return
3683 *  - On success, zero.
3684 *  - On failure, a negative value.
3685 */
3686static int
3687ixgbe_dev_macsec_interrupt_setup(struct rte_eth_dev *dev)
3688{
3689	struct ixgbe_interrupt *intr =
3690		IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
3691
3692	intr->mask |= IXGBE_EICR_LINKSEC;
3693
3694	return 0;
3695}
3696
3697/*
3698 * It reads ICR and sets flag (IXGBE_EICR_LSC) for the link_update.
3699 *
3700 * @param dev
3701 *  Pointer to struct rte_eth_dev.
3702 *
3703 * @return
3704 *  - On success, zero.
3705 *  - On failure, a negative value.
3706 */
3707static int
3708ixgbe_dev_interrupt_get_status(struct rte_eth_dev *dev)
3709{
3710	uint32_t eicr;
3711	struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3712	struct ixgbe_interrupt *intr =
3713		IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
3714
3715	/* clear all cause mask */
3716	ixgbe_disable_intr(hw);
3717
3718	/* read-on-clear nic registers here */
3719	eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
3720	PMD_DRV_LOG(DEBUG, "eicr %x", eicr);
3721
3722	intr->flags = 0;
3723
3724	/* set flag for async link update */
3725	if (eicr & IXGBE_EICR_LSC)
3726		intr->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
3727
3728	if (eicr & IXGBE_EICR_MAILBOX)
3729		intr->flags |= IXGBE_FLAG_MAILBOX;
3730
3731	if (eicr & IXGBE_EICR_LINKSEC)
3732		intr->flags |= IXGBE_FLAG_MACSEC;
3733
3734	if (hw->mac.type ==  ixgbe_mac_X550EM_x &&
3735	    hw->phy.type == ixgbe_phy_x550em_ext_t &&
3736	    (eicr & IXGBE_EICR_GPI_SDP0_X550EM_x))
3737		intr->flags |= IXGBE_FLAG_PHY_INTERRUPT;
3738
3739	return 0;
3740}
3741
3742/**
3743 * It gets and then prints the link status.
3744 *
3745 * @param dev
3746 *  Pointer to struct rte_eth_dev.
3747 *
3748 * @return
3749 *  - On success, zero.
3750 *  - On failure, a negative value.
3751 */
3752static void
3753ixgbe_dev_link_status_print(struct rte_eth_dev *dev)
3754{
3755	struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(dev);
3756	struct rte_eth_link link;
3757
3758	memset(&link, 0, sizeof(link));
3759	rte_ixgbe_dev_atomic_read_link_status(dev, &link);
3760	if (link.link_status) {
3761		PMD_INIT_LOG(INFO, "Port %d: Link Up - speed %u Mbps - %s",
3762					(int)(dev->data->port_id),
3763					(unsigned)link.link_speed,
3764			link.link_duplex == ETH_LINK_FULL_DUPLEX ?
3765					"full-duplex" : "half-duplex");
3766	} else {
3767		PMD_INIT_LOG(INFO, " Port %d: Link Down",
3768				(int)(dev->data->port_id));
3769	}
3770	PMD_INIT_LOG(DEBUG, "PCI Address: " PCI_PRI_FMT,
3771				pci_dev->addr.domain,
3772				pci_dev->addr.bus,
3773				pci_dev->addr.devid,
3774				pci_dev->addr.function);
3775}
3776
3777/*
3778 * It executes link_update after knowing an interrupt occurred.
3779 *
3780 * @param dev
3781 *  Pointer to struct rte_eth_dev.
3782 *
3783 * @return
3784 *  - On success, zero.
3785 *  - On failure, a negative value.
3786 */
3787static int
3788ixgbe_dev_interrupt_action(struct rte_eth_dev *dev,
3789			   struct rte_intr_handle *intr_handle)
3790{
3791	struct ixgbe_interrupt *intr =
3792		IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
3793	int64_t timeout;
3794	struct rte_eth_link link;
3795	struct ixgbe_hw *hw =
3796		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3797
3798	PMD_DRV_LOG(DEBUG, "intr action type %d", intr->flags);
3799
3800	if (intr->flags & IXGBE_FLAG_MAILBOX) {
3801		ixgbe_pf_mbx_process(dev);
3802		intr->flags &= ~IXGBE_FLAG_MAILBOX;
3803	}
3804
3805	if (intr->flags & IXGBE_FLAG_PHY_INTERRUPT) {
3806		ixgbe_handle_lasi(hw);
3807		intr->flags &= ~IXGBE_FLAG_PHY_INTERRUPT;
3808	}
3809
3810	if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
3811		/* get the link status before link update, for predicting later */
3812		memset(&link, 0, sizeof(link));
3813		rte_ixgbe_dev_atomic_read_link_status(dev, &link);
3814
3815		ixgbe_dev_link_update(dev, 0);
3816
3817		/* likely to up */
3818		if (!link.link_status)
3819			/* handle it 1 sec later, wait it being stable */
3820			timeout = IXGBE_LINK_UP_CHECK_TIMEOUT;
3821		/* likely to down */
3822		else
3823			/* handle it 4 sec later, wait it being stable */
3824			timeout = IXGBE_LINK_DOWN_CHECK_TIMEOUT;
3825
3826		ixgbe_dev_link_status_print(dev);
3827		intr->mask_original = intr->mask;
3828		/* only disable lsc interrupt */
3829		intr->mask &= ~IXGBE_EIMS_LSC;
3830		if (rte_eal_alarm_set(timeout * 1000,
3831				      ixgbe_dev_interrupt_delayed_handler, (void *)dev) < 0)
3832			PMD_DRV_LOG(ERR, "Error setting alarm");
3833		else
3834			intr->mask = intr->mask_original;
3835	}
3836
3837	PMD_DRV_LOG(DEBUG, "enable intr immediately");
3838	ixgbe_enable_intr(dev);
3839	rte_intr_enable(intr_handle);
3840
3841	return 0;
3842}
3843
3844/**
3845 * Interrupt handler which shall be registered for alarm callback for delayed
3846 * handling specific interrupt to wait for the stable nic state. As the
3847 * NIC interrupt state is not stable for ixgbe after link is just down,
3848 * it needs to wait 4 seconds to get the stable status.
3849 *
3850 * @param handle
3851 *  Pointer to interrupt handle.
3852 * @param param
3853 *  The address of parameter (struct rte_eth_dev *) regsitered before.
3854 *
3855 * @return
3856 *  void
3857 */
3858static void
3859ixgbe_dev_interrupt_delayed_handler(void *param)
3860{
3861	struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
3862	struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(dev);
3863	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
3864	struct ixgbe_interrupt *intr =
3865		IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
3866	struct ixgbe_hw *hw =
3867		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3868	uint32_t eicr;
3869
3870	ixgbe_disable_intr(hw);
3871
3872	eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
3873	if (eicr & IXGBE_EICR_MAILBOX)
3874		ixgbe_pf_mbx_process(dev);
3875
3876	if (intr->flags & IXGBE_FLAG_PHY_INTERRUPT) {
3877		ixgbe_handle_lasi(hw);
3878		intr->flags &= ~IXGBE_FLAG_PHY_INTERRUPT;
3879	}
3880
3881	if (intr->flags & IXGBE_FLAG_NEED_LINK_UPDATE) {
3882		ixgbe_dev_link_update(dev, 0);
3883		intr->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE;
3884		ixgbe_dev_link_status_print(dev);
3885		_rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
3886	}
3887
3888	if (intr->flags & IXGBE_FLAG_MACSEC) {
3889		_rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_MACSEC,
3890					      NULL);
3891		intr->flags &= ~IXGBE_FLAG_MACSEC;
3892	}
3893
3894	/* restore original mask */
3895	intr->mask = intr->mask_original;
3896	intr->mask_original = 0;
3897
3898	PMD_DRV_LOG(DEBUG, "enable intr in delayed handler S[%08x]", eicr);
3899	ixgbe_enable_intr(dev);
3900	rte_intr_enable(intr_handle);
3901}
3902
3903/**
3904 * Interrupt handler triggered by NIC  for handling
3905 * specific interrupt.
3906 *
3907 * @param handle
3908 *  Pointer to interrupt handle.
3909 * @param param
3910 *  The address of parameter (struct rte_eth_dev *) regsitered before.
3911 *
3912 * @return
3913 *  void
3914 */
3915static void
3916ixgbe_dev_interrupt_handler(struct rte_intr_handle *handle,
3917			    void *param)
3918{
3919	struct rte_eth_dev *dev = (struct rte_eth_dev *)param;
3920
3921	ixgbe_dev_interrupt_get_status(dev);
3922	ixgbe_dev_interrupt_action(dev, handle);
3923}
3924
3925static int
3926ixgbe_dev_led_on(struct rte_eth_dev *dev)
3927{
3928	struct ixgbe_hw *hw;
3929
3930	hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3931	return ixgbe_led_on(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP;
3932}
3933
3934static int
3935ixgbe_dev_led_off(struct rte_eth_dev *dev)
3936{
3937	struct ixgbe_hw *hw;
3938
3939	hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3940	return ixgbe_led_off(hw, 0) == IXGBE_SUCCESS ? 0 : -ENOTSUP;
3941}
3942
3943static int
3944ixgbe_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
3945{
3946	struct ixgbe_hw *hw;
3947	uint32_t mflcn_reg;
3948	uint32_t fccfg_reg;
3949	int rx_pause;
3950	int tx_pause;
3951
3952	hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
3953
3954	fc_conf->pause_time = hw->fc.pause_time;
3955	fc_conf->high_water = hw->fc.high_water[0];
3956	fc_conf->low_water = hw->fc.low_water[0];
3957	fc_conf->send_xon = hw->fc.send_xon;
3958	fc_conf->autoneg = !hw->fc.disable_fc_autoneg;
3959
3960	/*
3961	 * Return rx_pause status according to actual setting of
3962	 * MFLCN register.
3963	 */
3964	mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN);
3965	if (mflcn_reg & (IXGBE_MFLCN_RPFCE | IXGBE_MFLCN_RFCE))
3966		rx_pause = 1;
3967	else
3968		rx_pause = 0;
3969
3970	/*
3971	 * Return tx_pause status according to actual setting of
3972	 * FCCFG register.
3973	 */
3974	fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
3975	if (fccfg_reg & (IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY))
3976		tx_pause = 1;
3977	else
3978		tx_pause = 0;
3979
3980	if (rx_pause && tx_pause)
3981		fc_conf->mode = RTE_FC_FULL;
3982	else if (rx_pause)
3983		fc_conf->mode = RTE_FC_RX_PAUSE;
3984	else if (tx_pause)
3985		fc_conf->mode = RTE_FC_TX_PAUSE;
3986	else
3987		fc_conf->mode = RTE_FC_NONE;
3988
3989	return 0;
3990}
3991
3992static int
3993ixgbe_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
3994{
3995	struct ixgbe_hw *hw;
3996	int err;
3997	uint32_t rx_buf_size;
3998	uint32_t max_high_water;
3999	uint32_t mflcn;
4000	enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
4001		ixgbe_fc_none,
4002		ixgbe_fc_rx_pause,
4003		ixgbe_fc_tx_pause,
4004		ixgbe_fc_full
4005	};
4006
4007	PMD_INIT_FUNC_TRACE();
4008
4009	hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4010	rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(0));
4011	PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
4012
4013	/*
4014	 * At least reserve one Ethernet frame for watermark
4015	 * high_water/low_water in kilo bytes for ixgbe
4016	 */
4017	max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
4018	if ((fc_conf->high_water > max_high_water) ||
4019		(fc_conf->high_water < fc_conf->low_water)) {
4020		PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB");
4021		PMD_INIT_LOG(ERR, "High_water must <= 0x%x", max_high_water);
4022		return -EINVAL;
4023	}
4024
4025	hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[fc_conf->mode];
4026	hw->fc.pause_time     = fc_conf->pause_time;
4027	hw->fc.high_water[0]  = fc_conf->high_water;
4028	hw->fc.low_water[0]   = fc_conf->low_water;
4029	hw->fc.send_xon       = fc_conf->send_xon;
4030	hw->fc.disable_fc_autoneg = !fc_conf->autoneg;
4031
4032	err = ixgbe_fc_enable(hw);
4033
4034	/* Not negotiated is not an error case */
4035	if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED)) {
4036
4037		/* check if we want to forward MAC frames - driver doesn't have native
4038		 * capability to do that, so we'll write the registers ourselves */
4039
4040		mflcn = IXGBE_READ_REG(hw, IXGBE_MFLCN);
4041
4042		/* set or clear MFLCN.PMCF bit depending on configuration */
4043		if (fc_conf->mac_ctrl_frame_fwd != 0)
4044			mflcn |= IXGBE_MFLCN_PMCF;
4045		else
4046			mflcn &= ~IXGBE_MFLCN_PMCF;
4047
4048		IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn);
4049		IXGBE_WRITE_FLUSH(hw);
4050
4051		return 0;
4052	}
4053
4054	PMD_INIT_LOG(ERR, "ixgbe_fc_enable = 0x%x", err);
4055	return -EIO;
4056}
4057
4058/**
4059 *  ixgbe_pfc_enable_generic - Enable flow control
4060 *  @hw: pointer to hardware structure
4061 *  @tc_num: traffic class number
4062 *  Enable flow control according to the current settings.
4063 */
4064static int
4065ixgbe_dcb_pfc_enable_generic(struct ixgbe_hw *hw, uint8_t tc_num)
4066{
4067	int ret_val = 0;
4068	uint32_t mflcn_reg, fccfg_reg;
4069	uint32_t reg;
4070	uint32_t fcrtl, fcrth;
4071	uint8_t i;
4072	uint8_t nb_rx_en;
4073
4074	/* Validate the water mark configuration */
4075	if (!hw->fc.pause_time) {
4076		ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
4077		goto out;
4078	}
4079
4080	/* Low water mark of zero causes XOFF floods */
4081	if (hw->fc.current_mode & ixgbe_fc_tx_pause) {
4082		 /* High/Low water can not be 0 */
4083		if ((!hw->fc.high_water[tc_num]) || (!hw->fc.low_water[tc_num])) {
4084			PMD_INIT_LOG(ERR, "Invalid water mark configuration");
4085			ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
4086			goto out;
4087		}
4088
4089		if (hw->fc.low_water[tc_num] >= hw->fc.high_water[tc_num]) {
4090			PMD_INIT_LOG(ERR, "Invalid water mark configuration");
4091			ret_val = IXGBE_ERR_INVALID_LINK_SETTINGS;
4092			goto out;
4093		}
4094	}
4095	/* Negotiate the fc mode to use */
4096	ixgbe_fc_autoneg(hw);
4097
4098	/* Disable any previous flow control settings */
4099	mflcn_reg = IXGBE_READ_REG(hw, IXGBE_MFLCN);
4100	mflcn_reg &= ~(IXGBE_MFLCN_RPFCE_SHIFT | IXGBE_MFLCN_RFCE|IXGBE_MFLCN_RPFCE);
4101
4102	fccfg_reg = IXGBE_READ_REG(hw, IXGBE_FCCFG);
4103	fccfg_reg &= ~(IXGBE_FCCFG_TFCE_802_3X | IXGBE_FCCFG_TFCE_PRIORITY);
4104
4105	switch (hw->fc.current_mode) {
4106	case ixgbe_fc_none:
4107		/*
4108		 * If the count of enabled RX Priority Flow control >1,
4109		 * and the TX pause can not be disabled
4110		 */
4111		nb_rx_en = 0;
4112		for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
4113			reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
4114			if (reg & IXGBE_FCRTH_FCEN)
4115				nb_rx_en++;
4116		}
4117		if (nb_rx_en > 1)
4118			fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY;
4119		break;
4120	case ixgbe_fc_rx_pause:
4121		/*
4122		 * Rx Flow control is enabled and Tx Flow control is
4123		 * disabled by software override. Since there really
4124		 * isn't a way to advertise that we are capable of RX
4125		 * Pause ONLY, we will advertise that we support both
4126		 * symmetric and asymmetric Rx PAUSE.  Later, we will
4127		 * disable the adapter's ability to send PAUSE frames.
4128		 */
4129		mflcn_reg |= IXGBE_MFLCN_RPFCE;
4130		/*
4131		 * If the count of enabled RX Priority Flow control >1,
4132		 * and the TX pause can not be disabled
4133		 */
4134		nb_rx_en = 0;
4135		for (i = 0; i < IXGBE_DCB_MAX_TRAFFIC_CLASS; i++) {
4136			reg = IXGBE_READ_REG(hw, IXGBE_FCRTH_82599(i));
4137			if (reg & IXGBE_FCRTH_FCEN)
4138				nb_rx_en++;
4139		}
4140		if (nb_rx_en > 1)
4141			fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY;
4142		break;
4143	case ixgbe_fc_tx_pause:
4144		/*
4145		 * Tx Flow control is enabled, and Rx Flow control is
4146		 * disabled by software override.
4147		 */
4148		fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY;
4149		break;
4150	case ixgbe_fc_full:
4151		/* Flow control (both Rx and Tx) is enabled by SW override. */
4152		mflcn_reg |= IXGBE_MFLCN_RPFCE;
4153		fccfg_reg |= IXGBE_FCCFG_TFCE_PRIORITY;
4154		break;
4155	default:
4156		PMD_DRV_LOG(DEBUG, "Flow control param set incorrectly");
4157		ret_val = IXGBE_ERR_CONFIG;
4158		goto out;
4159	}
4160
4161	/* Set 802.3x based flow control settings. */
4162	mflcn_reg |= IXGBE_MFLCN_DPF;
4163	IXGBE_WRITE_REG(hw, IXGBE_MFLCN, mflcn_reg);
4164	IXGBE_WRITE_REG(hw, IXGBE_FCCFG, fccfg_reg);
4165
4166	/* Set up and enable Rx high/low water mark thresholds, enable XON. */
4167	if ((hw->fc.current_mode & ixgbe_fc_tx_pause) &&
4168		hw->fc.high_water[tc_num]) {
4169		fcrtl = (hw->fc.low_water[tc_num] << 10) | IXGBE_FCRTL_XONE;
4170		IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), fcrtl);
4171		fcrth = (hw->fc.high_water[tc_num] << 10) | IXGBE_FCRTH_FCEN;
4172	} else {
4173		IXGBE_WRITE_REG(hw, IXGBE_FCRTL_82599(tc_num), 0);
4174		/*
4175		 * In order to prevent Tx hangs when the internal Tx
4176		 * switch is enabled we must set the high water mark
4177		 * to the maximum FCRTH value.  This allows the Tx
4178		 * switch to function even under heavy Rx workloads.
4179		 */
4180		fcrth = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num)) - 32;
4181	}
4182	IXGBE_WRITE_REG(hw, IXGBE_FCRTH_82599(tc_num), fcrth);
4183
4184	/* Configure pause time (2 TCs per register) */
4185	reg = hw->fc.pause_time * 0x00010001;
4186	for (i = 0; i < (IXGBE_DCB_MAX_TRAFFIC_CLASS / 2); i++)
4187		IXGBE_WRITE_REG(hw, IXGBE_FCTTV(i), reg);
4188
4189	/* Configure flow control refresh threshold value */
4190	IXGBE_WRITE_REG(hw, IXGBE_FCRTV, hw->fc.pause_time / 2);
4191
4192out:
4193	return ret_val;
4194}
4195
4196static int
4197ixgbe_dcb_pfc_enable(struct rte_eth_dev *dev, uint8_t tc_num)
4198{
4199	struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4200	int32_t ret_val = IXGBE_NOT_IMPLEMENTED;
4201
4202	if (hw->mac.type != ixgbe_mac_82598EB) {
4203		ret_val = ixgbe_dcb_pfc_enable_generic(hw, tc_num);
4204	}
4205	return ret_val;
4206}
4207
4208static int
4209ixgbe_priority_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_pfc_conf *pfc_conf)
4210{
4211	int err;
4212	uint32_t rx_buf_size;
4213	uint32_t max_high_water;
4214	uint8_t tc_num;
4215	uint8_t  map[IXGBE_DCB_MAX_USER_PRIORITY] = { 0 };
4216	struct ixgbe_hw *hw =
4217		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4218	struct ixgbe_dcb_config *dcb_config =
4219		IXGBE_DEV_PRIVATE_TO_DCB_CFG(dev->data->dev_private);
4220
4221	enum ixgbe_fc_mode rte_fcmode_2_ixgbe_fcmode[] = {
4222		ixgbe_fc_none,
4223		ixgbe_fc_rx_pause,
4224		ixgbe_fc_tx_pause,
4225		ixgbe_fc_full
4226	};
4227
4228	PMD_INIT_FUNC_TRACE();
4229
4230	ixgbe_dcb_unpack_map_cee(dcb_config, IXGBE_DCB_RX_CONFIG, map);
4231	tc_num = map[pfc_conf->priority];
4232	rx_buf_size = IXGBE_READ_REG(hw, IXGBE_RXPBSIZE(tc_num));
4233	PMD_INIT_LOG(DEBUG, "Rx packet buffer size = 0x%x", rx_buf_size);
4234	/*
4235	 * At least reserve one Ethernet frame for watermark
4236	 * high_water/low_water in kilo bytes for ixgbe
4237	 */
4238	max_high_water = (rx_buf_size - ETHER_MAX_LEN) >> IXGBE_RXPBSIZE_SHIFT;
4239	if ((pfc_conf->fc.high_water > max_high_water) ||
4240	    (pfc_conf->fc.high_water <= pfc_conf->fc.low_water)) {
4241		PMD_INIT_LOG(ERR, "Invalid high/low water setup value in KB");
4242		PMD_INIT_LOG(ERR, "High_water must <= 0x%x", max_high_water);
4243		return -EINVAL;
4244	}
4245
4246	hw->fc.requested_mode = rte_fcmode_2_ixgbe_fcmode[pfc_conf->fc.mode];
4247	hw->fc.pause_time = pfc_conf->fc.pause_time;
4248	hw->fc.send_xon = pfc_conf->fc.send_xon;
4249	hw->fc.low_water[tc_num] =  pfc_conf->fc.low_water;
4250	hw->fc.high_water[tc_num] = pfc_conf->fc.high_water;
4251
4252	err = ixgbe_dcb_pfc_enable(dev, tc_num);
4253
4254	/* Not negotiated is not an error case */
4255	if ((err == IXGBE_SUCCESS) || (err == IXGBE_ERR_FC_NOT_NEGOTIATED))
4256		return 0;
4257
4258	PMD_INIT_LOG(ERR, "ixgbe_dcb_pfc_enable = 0x%x", err);
4259	return -EIO;
4260}
4261
4262static int
4263ixgbe_dev_rss_reta_update(struct rte_eth_dev *dev,
4264			  struct rte_eth_rss_reta_entry64 *reta_conf,
4265			  uint16_t reta_size)
4266{
4267	uint16_t i, sp_reta_size;
4268	uint8_t j, mask;
4269	uint32_t reta, r;
4270	uint16_t idx, shift;
4271	struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4272	uint32_t reta_reg;
4273
4274	PMD_INIT_FUNC_TRACE();
4275
4276	if (!ixgbe_rss_update_sp(hw->mac.type)) {
4277		PMD_DRV_LOG(ERR, "RSS reta update is not supported on this "
4278			"NIC.");
4279		return -ENOTSUP;
4280	}
4281
4282	sp_reta_size = ixgbe_reta_size_get(hw->mac.type);
4283	if (reta_size != sp_reta_size) {
4284		PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
4285			"(%d) doesn't match the number hardware can supported "
4286			"(%d)", reta_size, sp_reta_size);
4287		return -EINVAL;
4288	}
4289
4290	for (i = 0; i < reta_size; i += IXGBE_4_BIT_WIDTH) {
4291		idx = i / RTE_RETA_GROUP_SIZE;
4292		shift = i % RTE_RETA_GROUP_SIZE;
4293		mask = (uint8_t)((reta_conf[idx].mask >> shift) &
4294						IXGBE_4_BIT_MASK);
4295		if (!mask)
4296			continue;
4297		reta_reg = ixgbe_reta_reg_get(hw->mac.type, i);
4298		if (mask == IXGBE_4_BIT_MASK)
4299			r = 0;
4300		else
4301			r = IXGBE_READ_REG(hw, reta_reg);
4302		for (j = 0, reta = 0; j < IXGBE_4_BIT_WIDTH; j++) {
4303			if (mask & (0x1 << j))
4304				reta |= reta_conf[idx].reta[shift + j] <<
4305							(CHAR_BIT * j);
4306			else
4307				reta |= r & (IXGBE_8_BIT_MASK <<
4308						(CHAR_BIT * j));
4309		}
4310		IXGBE_WRITE_REG(hw, reta_reg, reta);
4311	}
4312
4313	return 0;
4314}
4315
4316static int
4317ixgbe_dev_rss_reta_query(struct rte_eth_dev *dev,
4318			 struct rte_eth_rss_reta_entry64 *reta_conf,
4319			 uint16_t reta_size)
4320{
4321	uint16_t i, sp_reta_size;
4322	uint8_t j, mask;
4323	uint32_t reta;
4324	uint16_t idx, shift;
4325	struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4326	uint32_t reta_reg;
4327
4328	PMD_INIT_FUNC_TRACE();
4329	sp_reta_size = ixgbe_reta_size_get(hw->mac.type);
4330	if (reta_size != sp_reta_size) {
4331		PMD_DRV_LOG(ERR, "The size of hash lookup table configured "
4332			"(%d) doesn't match the number hardware can supported "
4333			"(%d)", reta_size, sp_reta_size);
4334		return -EINVAL;
4335	}
4336
4337	for (i = 0; i < reta_size; i += IXGBE_4_BIT_WIDTH) {
4338		idx = i / RTE_RETA_GROUP_SIZE;
4339		shift = i % RTE_RETA_GROUP_SIZE;
4340		mask = (uint8_t)((reta_conf[idx].mask >> shift) &
4341						IXGBE_4_BIT_MASK);
4342		if (!mask)
4343			continue;
4344
4345		reta_reg = ixgbe_reta_reg_get(hw->mac.type, i);
4346		reta = IXGBE_READ_REG(hw, reta_reg);
4347		for (j = 0; j < IXGBE_4_BIT_WIDTH; j++) {
4348			if (mask & (0x1 << j))
4349				reta_conf[idx].reta[shift + j] =
4350					((reta >> (CHAR_BIT * j)) &
4351						IXGBE_8_BIT_MASK);
4352		}
4353	}
4354
4355	return 0;
4356}
4357
4358static void
4359ixgbe_add_rar(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
4360				uint32_t index, uint32_t pool)
4361{
4362	struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4363	uint32_t enable_addr = 1;
4364
4365	ixgbe_set_rar(hw, index, mac_addr->addr_bytes, pool, enable_addr);
4366}
4367
4368static void
4369ixgbe_remove_rar(struct rte_eth_dev *dev, uint32_t index)
4370{
4371	struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4372
4373	ixgbe_clear_rar(hw, index);
4374}
4375
4376static void
4377ixgbe_set_default_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr)
4378{
4379	ixgbe_remove_rar(dev, 0);
4380
4381	ixgbe_add_rar(dev, addr, 0, 0);
4382}
4383
4384static bool
4385is_device_supported(struct rte_eth_dev *dev, struct eth_driver *drv)
4386{
4387	if (strcmp(dev->driver->pci_drv.driver.name,
4388		   drv->pci_drv.driver.name))
4389		return false;
4390
4391	return true;
4392}
4393
4394int
4395rte_pmd_ixgbe_set_vf_mac_addr(uint8_t port, uint16_t vf,
4396		struct ether_addr *mac_addr)
4397{
4398	struct ixgbe_hw *hw;
4399	struct ixgbe_vf_info *vfinfo;
4400	int rar_entry;
4401	uint8_t *new_mac = (uint8_t *)(mac_addr);
4402	struct rte_eth_dev *dev;
4403	struct rte_pci_device *pci_dev;
4404
4405	RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV);
4406
4407	dev = &rte_eth_devices[port];
4408	pci_dev = IXGBE_DEV_TO_PCI(dev);
4409
4410	if (!is_device_supported(dev, &rte_ixgbe_pmd))
4411		return -ENOTSUP;
4412
4413	if (vf >= pci_dev->max_vfs)
4414		return -EINVAL;
4415
4416	hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4417	vfinfo = *(IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private));
4418	rar_entry = hw->mac.num_rar_entries - (vf + 1);
4419
4420	if (is_valid_assigned_ether_addr((struct ether_addr *)new_mac)) {
4421		rte_memcpy(vfinfo[vf].vf_mac_addresses, new_mac,
4422				ETHER_ADDR_LEN);
4423		return hw->mac.ops.set_rar(hw, rar_entry, new_mac, vf,
4424				IXGBE_RAH_AV);
4425	}
4426	return -EINVAL;
4427}
4428
4429static int
4430ixgbe_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
4431{
4432	uint32_t hlreg0;
4433	uint32_t maxfrs;
4434	struct ixgbe_hw *hw;
4435	struct rte_eth_dev_info dev_info;
4436	uint32_t frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN;
4437
4438	ixgbe_dev_info_get(dev, &dev_info);
4439
4440	/* check that mtu is within the allowed range */
4441	if ((mtu < ETHER_MIN_MTU) || (frame_size > dev_info.max_rx_pktlen))
4442		return -EINVAL;
4443
4444	/* refuse mtu that requires the support of scattered packets when this
4445	 * feature has not been enabled before.
4446	 */
4447	if (!dev->data->scattered_rx &&
4448	    (frame_size + 2 * IXGBE_VLAN_TAG_SIZE >
4449	     dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM))
4450		return -EINVAL;
4451
4452	hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4453	hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
4454
4455	/* switch to jumbo mode if needed */
4456	if (frame_size > ETHER_MAX_LEN) {
4457		dev->data->dev_conf.rxmode.jumbo_frame = 1;
4458		hlreg0 |= IXGBE_HLREG0_JUMBOEN;
4459	} else {
4460		dev->data->dev_conf.rxmode.jumbo_frame = 0;
4461		hlreg0 &= ~IXGBE_HLREG0_JUMBOEN;
4462	}
4463	IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
4464
4465	/* update max frame size */
4466	dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
4467
4468	maxfrs = IXGBE_READ_REG(hw, IXGBE_MAXFRS);
4469	maxfrs &= 0x0000FFFF;
4470	maxfrs |= (dev->data->dev_conf.rxmode.max_rx_pkt_len << 16);
4471	IXGBE_WRITE_REG(hw, IXGBE_MAXFRS, maxfrs);
4472
4473	return 0;
4474}
4475
4476/*
4477 * Virtual Function operations
4478 */
4479static void
4480ixgbevf_intr_disable(struct ixgbe_hw *hw)
4481{
4482	PMD_INIT_FUNC_TRACE();
4483
4484	/* Clear interrupt mask to stop from interrupts being generated */
4485	IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, IXGBE_VF_IRQ_CLEAR_MASK);
4486
4487	IXGBE_WRITE_FLUSH(hw);
4488}
4489
4490static void
4491ixgbevf_intr_enable(struct ixgbe_hw *hw)
4492{
4493	PMD_INIT_FUNC_TRACE();
4494
4495	/* VF enable interrupt autoclean */
4496	IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, IXGBE_VF_IRQ_ENABLE_MASK);
4497	IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, IXGBE_VF_IRQ_ENABLE_MASK);
4498	IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, IXGBE_VF_IRQ_ENABLE_MASK);
4499
4500	IXGBE_WRITE_FLUSH(hw);
4501}
4502
4503static int
4504ixgbevf_dev_configure(struct rte_eth_dev *dev)
4505{
4506	struct rte_eth_conf *conf = &dev->data->dev_conf;
4507	struct ixgbe_adapter *adapter =
4508			(struct ixgbe_adapter *)dev->data->dev_private;
4509
4510	PMD_INIT_LOG(DEBUG, "Configured Virtual Function port id: %d",
4511		     dev->data->port_id);
4512
4513	/*
4514	 * VF has no ability to enable/disable HW CRC
4515	 * Keep the persistent behavior the same as Host PF
4516	 */
4517#ifndef RTE_LIBRTE_IXGBE_PF_DISABLE_STRIP_CRC
4518	if (!conf->rxmode.hw_strip_crc) {
4519		PMD_INIT_LOG(NOTICE, "VF can't disable HW CRC Strip");
4520		conf->rxmode.hw_strip_crc = 1;
4521	}
4522#else
4523	if (conf->rxmode.hw_strip_crc) {
4524		PMD_INIT_LOG(NOTICE, "VF can't enable HW CRC Strip");
4525		conf->rxmode.hw_strip_crc = 0;
4526	}
4527#endif
4528
4529	/*
4530	 * Initialize to TRUE. If any of Rx queues doesn't meet the bulk
4531	 * allocation or vector Rx preconditions we will reset it.
4532	 */
4533	adapter->rx_bulk_alloc_allowed = true;
4534	adapter->rx_vec_allowed = true;
4535
4536	return 0;
4537}
4538
4539static int
4540ixgbevf_dev_start(struct rte_eth_dev *dev)
4541{
4542	struct ixgbe_hw *hw =
4543		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4544	uint32_t intr_vector = 0;
4545	struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(dev);
4546	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
4547
4548	int err, mask = 0;
4549
4550	PMD_INIT_FUNC_TRACE();
4551
4552	hw->mac.ops.reset_hw(hw);
4553	hw->mac.get_link_status = true;
4554
4555	/* negotiate mailbox API version to use with the PF. */
4556	ixgbevf_negotiate_api(hw);
4557
4558	ixgbevf_dev_tx_init(dev);
4559
4560	/* This can fail when allocating mbufs for descriptor rings */
4561	err = ixgbevf_dev_rx_init(dev);
4562	if (err) {
4563		PMD_INIT_LOG(ERR, "Unable to initialize RX hardware (%d)", err);
4564		ixgbe_dev_clear_queues(dev);
4565		return err;
4566	}
4567
4568	/* Set vfta */
4569	ixgbevf_set_vfta_all(dev, 1);
4570
4571	/* Set HW strip */
4572	mask = ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK |
4573		ETH_VLAN_EXTEND_MASK;
4574	ixgbevf_vlan_offload_set(dev, mask);
4575
4576	ixgbevf_dev_rxtx_start(dev);
4577
4578	/* check and configure queue intr-vector mapping */
4579	if (dev->data->dev_conf.intr_conf.rxq != 0) {
4580		intr_vector = dev->data->nb_rx_queues;
4581		if (rte_intr_efd_enable(intr_handle, intr_vector))
4582			return -1;
4583	}
4584
4585	if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
4586		intr_handle->intr_vec =
4587			rte_zmalloc("intr_vec",
4588				    dev->data->nb_rx_queues * sizeof(int), 0);
4589		if (intr_handle->intr_vec == NULL) {
4590			PMD_INIT_LOG(ERR, "Failed to allocate %d rx_queues"
4591				     " intr_vec", dev->data->nb_rx_queues);
4592			return -ENOMEM;
4593		}
4594	}
4595	ixgbevf_configure_msix(dev);
4596
4597	rte_intr_enable(intr_handle);
4598
4599	/* Re-enable interrupt for VF */
4600	ixgbevf_intr_enable(hw);
4601
4602	return 0;
4603}
4604
4605static void
4606ixgbevf_dev_stop(struct rte_eth_dev *dev)
4607{
4608	struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4609	struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(dev);
4610	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
4611
4612	PMD_INIT_FUNC_TRACE();
4613
4614	ixgbevf_intr_disable(hw);
4615
4616	hw->adapter_stopped = 1;
4617	ixgbe_stop_adapter(hw);
4618
4619	/*
4620	  * Clear what we set, but we still keep shadow_vfta to
4621	  * restore after device starts
4622	  */
4623	ixgbevf_set_vfta_all(dev, 0);
4624
4625	/* Clear stored conf */
4626	dev->data->scattered_rx = 0;
4627
4628	ixgbe_dev_clear_queues(dev);
4629
4630	/* Clean datapath event and queue/vec mapping */
4631	rte_intr_efd_disable(intr_handle);
4632	if (intr_handle->intr_vec != NULL) {
4633		rte_free(intr_handle->intr_vec);
4634		intr_handle->intr_vec = NULL;
4635	}
4636}
4637
4638static void
4639ixgbevf_dev_close(struct rte_eth_dev *dev)
4640{
4641	struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4642
4643	PMD_INIT_FUNC_TRACE();
4644
4645	ixgbe_reset_hw(hw);
4646
4647	ixgbevf_dev_stop(dev);
4648
4649	ixgbe_dev_free_queues(dev);
4650
4651	/**
4652	 * Remove the VF MAC address ro ensure
4653	 * that the VF traffic goes to the PF
4654	 * after stop, close and detach of the VF
4655	 **/
4656	ixgbevf_remove_mac_addr(dev, 0);
4657}
4658
4659static void ixgbevf_set_vfta_all(struct rte_eth_dev *dev, bool on)
4660{
4661	struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4662	struct ixgbe_vfta *shadow_vfta =
4663		IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
4664	int i = 0, j = 0, vfta = 0, mask = 1;
4665
4666	for (i = 0; i < IXGBE_VFTA_SIZE; i++) {
4667		vfta = shadow_vfta->vfta[i];
4668		if (vfta) {
4669			mask = 1;
4670			for (j = 0; j < 32; j++) {
4671				if (vfta & mask)
4672					ixgbe_set_vfta(hw, (i<<5)+j, 0,
4673						       on, false);
4674				mask <<= 1;
4675			}
4676		}
4677	}
4678
4679}
4680
4681static int
4682ixgbevf_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
4683{
4684	struct ixgbe_hw *hw =
4685		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4686	struct ixgbe_vfta *shadow_vfta =
4687		IXGBE_DEV_PRIVATE_TO_VFTA(dev->data->dev_private);
4688	uint32_t vid_idx = 0;
4689	uint32_t vid_bit = 0;
4690	int ret = 0;
4691
4692	PMD_INIT_FUNC_TRACE();
4693
4694	/* vind is not used in VF driver, set to 0, check ixgbe_set_vfta_vf */
4695	ret = ixgbe_set_vfta(hw, vlan_id, 0, !!on, false);
4696	if (ret) {
4697		PMD_INIT_LOG(ERR, "Unable to set VF vlan");
4698		return ret;
4699	}
4700	vid_idx = (uint32_t) ((vlan_id >> 5) & 0x7F);
4701	vid_bit = (uint32_t) (1 << (vlan_id & 0x1F));
4702
4703	/* Save what we set and retore it after device reset */
4704	if (on)
4705		shadow_vfta->vfta[vid_idx] |= vid_bit;
4706	else
4707		shadow_vfta->vfta[vid_idx] &= ~vid_bit;
4708
4709	return 0;
4710}
4711
4712static void
4713ixgbevf_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
4714{
4715	struct ixgbe_hw *hw =
4716		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4717	uint32_t ctrl;
4718
4719	PMD_INIT_FUNC_TRACE();
4720
4721	if (queue >= hw->mac.max_rx_queues)
4722		return;
4723
4724	ctrl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(queue));
4725	if (on)
4726		ctrl |= IXGBE_RXDCTL_VME;
4727	else
4728		ctrl &= ~IXGBE_RXDCTL_VME;
4729	IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(queue), ctrl);
4730
4731	ixgbe_vlan_hw_strip_bitmap_set(dev, queue, on);
4732}
4733
4734static void
4735ixgbevf_vlan_offload_set(struct rte_eth_dev *dev, int mask)
4736{
4737	struct ixgbe_hw *hw =
4738		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4739	uint16_t i;
4740	int on = 0;
4741
4742	/* VF function only support hw strip feature, others are not support */
4743	if (mask & ETH_VLAN_STRIP_MASK) {
4744		on = !!(dev->data->dev_conf.rxmode.hw_vlan_strip);
4745
4746		for (i = 0; i < hw->mac.max_rx_queues; i++)
4747			ixgbevf_vlan_strip_queue_set(dev, i, on);
4748	}
4749}
4750
4751static int
4752ixgbe_vt_check(struct ixgbe_hw *hw)
4753{
4754	uint32_t reg_val;
4755
4756	/* if Virtualization Technology is enabled */
4757	reg_val = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
4758	if (!(reg_val & IXGBE_VT_CTL_VT_ENABLE)) {
4759		PMD_INIT_LOG(ERR, "VT must be enabled for this setting");
4760		return -1;
4761	}
4762
4763	return 0;
4764}
4765
4766static uint32_t
4767ixgbe_uta_vector(struct ixgbe_hw *hw, struct ether_addr *uc_addr)
4768{
4769	uint32_t vector = 0;
4770
4771	switch (hw->mac.mc_filter_type) {
4772	case 0:   /* use bits [47:36] of the address */
4773		vector = ((uc_addr->addr_bytes[4] >> 4) |
4774			(((uint16_t)uc_addr->addr_bytes[5]) << 4));
4775		break;
4776	case 1:   /* use bits [46:35] of the address */
4777		vector = ((uc_addr->addr_bytes[4] >> 3) |
4778			(((uint16_t)uc_addr->addr_bytes[5]) << 5));
4779		break;
4780	case 2:   /* use bits [45:34] of the address */
4781		vector = ((uc_addr->addr_bytes[4] >> 2) |
4782			(((uint16_t)uc_addr->addr_bytes[5]) << 6));
4783		break;
4784	case 3:   /* use bits [43:32] of the address */
4785		vector = ((uc_addr->addr_bytes[4]) |
4786			(((uint16_t)uc_addr->addr_bytes[5]) << 8));
4787		break;
4788	default:  /* Invalid mc_filter_type */
4789		break;
4790	}
4791
4792	/* vector can only be 12-bits or boundary will be exceeded */
4793	vector &= 0xFFF;
4794	return vector;
4795}
4796
4797static int
4798ixgbe_uc_hash_table_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
4799			uint8_t on)
4800{
4801	uint32_t vector;
4802	uint32_t uta_idx;
4803	uint32_t reg_val;
4804	uint32_t uta_shift;
4805	uint32_t rc;
4806	const uint32_t ixgbe_uta_idx_mask = 0x7F;
4807	const uint32_t ixgbe_uta_bit_shift = 5;
4808	const uint32_t ixgbe_uta_bit_mask = (0x1 << ixgbe_uta_bit_shift) - 1;
4809	const uint32_t bit1 = 0x1;
4810
4811	struct ixgbe_hw *hw =
4812		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4813	struct ixgbe_uta_info *uta_info =
4814		IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private);
4815
4816	/* The UTA table only exists on 82599 hardware and newer */
4817	if (hw->mac.type < ixgbe_mac_82599EB)
4818		return -ENOTSUP;
4819
4820	vector = ixgbe_uta_vector(hw, mac_addr);
4821	uta_idx = (vector >> ixgbe_uta_bit_shift) & ixgbe_uta_idx_mask;
4822	uta_shift = vector & ixgbe_uta_bit_mask;
4823
4824	rc = ((uta_info->uta_shadow[uta_idx] >> uta_shift & bit1) != 0);
4825	if (rc == on)
4826		return 0;
4827
4828	reg_val = IXGBE_READ_REG(hw, IXGBE_UTA(uta_idx));
4829	if (on) {
4830		uta_info->uta_in_use++;
4831		reg_val |= (bit1 << uta_shift);
4832		uta_info->uta_shadow[uta_idx] |= (bit1 << uta_shift);
4833	} else {
4834		uta_info->uta_in_use--;
4835		reg_val &= ~(bit1 << uta_shift);
4836		uta_info->uta_shadow[uta_idx] &= ~(bit1 << uta_shift);
4837	}
4838
4839	IXGBE_WRITE_REG(hw, IXGBE_UTA(uta_idx), reg_val);
4840
4841	if (uta_info->uta_in_use > 0)
4842		IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL,
4843				IXGBE_MCSTCTRL_MFE | hw->mac.mc_filter_type);
4844	else
4845		IXGBE_WRITE_REG(hw, IXGBE_MCSTCTRL, hw->mac.mc_filter_type);
4846
4847	return 0;
4848}
4849
4850static int
4851ixgbe_uc_all_hash_table_set(struct rte_eth_dev *dev, uint8_t on)
4852{
4853	int i;
4854	struct ixgbe_hw *hw =
4855		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4856	struct ixgbe_uta_info *uta_info =
4857		IXGBE_DEV_PRIVATE_TO_UTA(dev->data->dev_private);
4858
4859	/* The UTA table only exists on 82599 hardware and newer */
4860	if (hw->mac.type < ixgbe_mac_82599EB)
4861		return -ENOTSUP;
4862
4863	if (on) {
4864		for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
4865			uta_info->uta_shadow[i] = ~0;
4866			IXGBE_WRITE_REG(hw, IXGBE_UTA(i), ~0);
4867		}
4868	} else {
4869		for (i = 0; i < ETH_VMDQ_NUM_UC_HASH_ARRAY; i++) {
4870			uta_info->uta_shadow[i] = 0;
4871			IXGBE_WRITE_REG(hw, IXGBE_UTA(i), 0);
4872		}
4873	}
4874	return 0;
4875
4876}
4877
4878uint32_t
4879ixgbe_convert_vm_rx_mask_to_val(uint16_t rx_mask, uint32_t orig_val)
4880{
4881	uint32_t new_val = orig_val;
4882
4883	if (rx_mask & ETH_VMDQ_ACCEPT_UNTAG)
4884		new_val |= IXGBE_VMOLR_AUPE;
4885	if (rx_mask & ETH_VMDQ_ACCEPT_HASH_MC)
4886		new_val |= IXGBE_VMOLR_ROMPE;
4887	if (rx_mask & ETH_VMDQ_ACCEPT_HASH_UC)
4888		new_val |= IXGBE_VMOLR_ROPE;
4889	if (rx_mask & ETH_VMDQ_ACCEPT_BROADCAST)
4890		new_val |= IXGBE_VMOLR_BAM;
4891	if (rx_mask & ETH_VMDQ_ACCEPT_MULTICAST)
4892		new_val |= IXGBE_VMOLR_MPE;
4893
4894	return new_val;
4895}
4896
4897
4898int
4899rte_pmd_ixgbe_set_vf_vlan_anti_spoof(uint8_t port, uint16_t vf, uint8_t on)
4900{
4901	struct ixgbe_hw *hw;
4902	struct ixgbe_mac_info *mac;
4903	struct rte_eth_dev *dev;
4904	struct rte_pci_device *pci_dev;
4905
4906	RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV);
4907
4908	dev = &rte_eth_devices[port];
4909	pci_dev = IXGBE_DEV_TO_PCI(dev);
4910
4911	if (!is_device_supported(dev, &rte_ixgbe_pmd))
4912		return -ENOTSUP;
4913
4914	if (vf >= pci_dev->max_vfs)
4915		return -EINVAL;
4916
4917	if (on > 1)
4918		return -EINVAL;
4919
4920	hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4921	mac = &hw->mac;
4922
4923	mac->ops.set_vlan_anti_spoofing(hw, on, vf);
4924
4925	return 0;
4926}
4927
4928int
4929rte_pmd_ixgbe_set_vf_mac_anti_spoof(uint8_t port, uint16_t vf, uint8_t on)
4930{
4931	struct ixgbe_hw *hw;
4932	struct ixgbe_mac_info *mac;
4933	struct rte_eth_dev *dev;
4934	struct rte_pci_device *pci_dev;
4935
4936	RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV);
4937
4938	dev = &rte_eth_devices[port];
4939	pci_dev = IXGBE_DEV_TO_PCI(dev);
4940
4941	if (!is_device_supported(dev, &rte_ixgbe_pmd))
4942		return -ENOTSUP;
4943
4944	if (vf >= pci_dev->max_vfs)
4945		return -EINVAL;
4946
4947	if (on > 1)
4948		return -EINVAL;
4949
4950	hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4951	mac = &hw->mac;
4952	mac->ops.set_mac_anti_spoofing(hw, on, vf);
4953
4954	return 0;
4955}
4956
4957int
4958rte_pmd_ixgbe_set_vf_vlan_insert(uint8_t port, uint16_t vf, uint16_t vlan_id)
4959{
4960	struct ixgbe_hw *hw;
4961	uint32_t ctrl;
4962	struct rte_eth_dev *dev;
4963	struct rte_pci_device *pci_dev;
4964
4965	RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV);
4966
4967	dev = &rte_eth_devices[port];
4968	pci_dev = IXGBE_DEV_TO_PCI(dev);
4969
4970	if (!is_device_supported(dev, &rte_ixgbe_pmd))
4971		return -ENOTSUP;
4972
4973	if (vf >= pci_dev->max_vfs)
4974		return -EINVAL;
4975
4976	if (vlan_id > ETHER_MAX_VLAN_ID)
4977		return -EINVAL;
4978
4979	hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
4980	ctrl = IXGBE_READ_REG(hw, IXGBE_VMVIR(vf));
4981	if (vlan_id) {
4982		ctrl = vlan_id;
4983		ctrl |= IXGBE_VMVIR_VLANA_DEFAULT;
4984	} else {
4985		ctrl = 0;
4986	}
4987
4988	IXGBE_WRITE_REG(hw, IXGBE_VMVIR(vf), ctrl);
4989
4990	return 0;
4991}
4992
4993int
4994rte_pmd_ixgbe_set_tx_loopback(uint8_t port, uint8_t on)
4995{
4996	struct ixgbe_hw *hw;
4997	uint32_t ctrl;
4998	struct rte_eth_dev *dev;
4999
5000	RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV);
5001
5002	dev = &rte_eth_devices[port];
5003
5004	if (!is_device_supported(dev, &rte_ixgbe_pmd))
5005		return -ENOTSUP;
5006
5007	if (on > 1)
5008		return -EINVAL;
5009
5010	hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5011	ctrl = IXGBE_READ_REG(hw, IXGBE_PFDTXGSWC);
5012	/* enable or disable VMDQ loopback */
5013	if (on)
5014		ctrl |= IXGBE_PFDTXGSWC_VT_LBEN;
5015	else
5016		ctrl &= ~IXGBE_PFDTXGSWC_VT_LBEN;
5017
5018	IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, ctrl);
5019
5020	return 0;
5021}
5022
5023int
5024rte_pmd_ixgbe_set_all_queues_drop_en(uint8_t port, uint8_t on)
5025{
5026	struct ixgbe_hw *hw;
5027	uint32_t reg_value;
5028	int i;
5029	int num_queues = (int)(IXGBE_QDE_IDX_MASK >> IXGBE_QDE_IDX_SHIFT);
5030	struct rte_eth_dev *dev;
5031
5032	RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV);
5033
5034	dev = &rte_eth_devices[port];
5035
5036	if (!is_device_supported(dev, &rte_ixgbe_pmd))
5037		return -ENOTSUP;
5038
5039	if (on > 1)
5040		return -EINVAL;
5041
5042	hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5043	for (i = 0; i <= num_queues; i++) {
5044		reg_value = IXGBE_QDE_WRITE |
5045				(i << IXGBE_QDE_IDX_SHIFT) |
5046				(on & IXGBE_QDE_ENABLE);
5047		IXGBE_WRITE_REG(hw, IXGBE_QDE, reg_value);
5048	}
5049
5050	return 0;
5051}
5052
5053int
5054rte_pmd_ixgbe_set_vf_split_drop_en(uint8_t port, uint16_t vf, uint8_t on)
5055{
5056	struct ixgbe_hw *hw;
5057	uint32_t reg_value;
5058	struct rte_eth_dev *dev;
5059	struct rte_pci_device *pci_dev;
5060
5061	RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV);
5062
5063	dev = &rte_eth_devices[port];
5064	pci_dev = IXGBE_DEV_TO_PCI(dev);
5065
5066	if (!is_device_supported(dev, &rte_ixgbe_pmd))
5067		return -ENOTSUP;
5068
5069	/* only support VF's 0 to 63 */
5070	if ((vf >= pci_dev->max_vfs) || (vf > 63))
5071		return -EINVAL;
5072
5073	if (on > 1)
5074		return -EINVAL;
5075
5076	hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5077	reg_value = IXGBE_READ_REG(hw, IXGBE_SRRCTL(vf));
5078	if (on)
5079		reg_value |= IXGBE_SRRCTL_DROP_EN;
5080	else
5081		reg_value &= ~IXGBE_SRRCTL_DROP_EN;
5082
5083	IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(vf), reg_value);
5084
5085	return 0;
5086}
5087
5088int
5089rte_pmd_ixgbe_set_vf_vlan_stripq(uint8_t port, uint16_t vf, uint8_t on)
5090{
5091	struct rte_eth_dev *dev;
5092	struct rte_pci_device *pci_dev;
5093	struct ixgbe_hw *hw;
5094	uint16_t queues_per_pool;
5095	uint32_t q;
5096
5097	RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV);
5098
5099	dev = &rte_eth_devices[port];
5100	pci_dev = IXGBE_DEV_TO_PCI(dev);
5101	hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5102
5103	if (!is_device_supported(dev, &rte_ixgbe_pmd))
5104		return -ENOTSUP;
5105
5106	if (vf >= pci_dev->max_vfs)
5107		return -EINVAL;
5108
5109	if (on > 1)
5110		return -EINVAL;
5111
5112	RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_strip_queue_set, -ENOTSUP);
5113
5114	/* The PF has 128 queue pairs and in SRIOV configuration
5115	 * those queues will be assigned to VF's, so RXDCTL
5116	 * registers will be dealing with queues which will be
5117	 * assigned to VF's.
5118	 * Let's say we have SRIOV configured with 31 VF's then the
5119	 * first 124 queues 0-123 will be allocated to VF's and only
5120	 * the last 4 queues 123-127 will be assigned to the PF.
5121	 */
5122	if (hw->mac.type == ixgbe_mac_82598EB)
5123		queues_per_pool = (uint16_t)hw->mac.max_rx_queues /
5124				  ETH_16_POOLS;
5125	else
5126		queues_per_pool = (uint16_t)hw->mac.max_rx_queues /
5127				  ETH_64_POOLS;
5128
5129	for (q = 0; q < queues_per_pool; q++)
5130		(*dev->dev_ops->vlan_strip_queue_set)(dev,
5131				q + vf * queues_per_pool, on);
5132	return 0;
5133}
5134
5135int
5136rte_pmd_ixgbe_set_vf_rxmode(uint8_t port, uint16_t vf, uint16_t rx_mask, uint8_t on)
5137{
5138	int val = 0;
5139	struct rte_eth_dev *dev;
5140	struct rte_pci_device *pci_dev;
5141	struct ixgbe_hw *hw;
5142	uint32_t vmolr;
5143
5144	RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV);
5145
5146	dev = &rte_eth_devices[port];
5147	pci_dev = IXGBE_DEV_TO_PCI(dev);
5148
5149	if (!is_device_supported(dev, &rte_ixgbe_pmd))
5150		return -ENOTSUP;
5151
5152	if (vf >= pci_dev->max_vfs)
5153		return -EINVAL;
5154
5155	if (on > 1)
5156		return -EINVAL;
5157
5158	hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5159	vmolr = IXGBE_READ_REG(hw, IXGBE_VMOLR(vf));
5160
5161	if (hw->mac.type == ixgbe_mac_82598EB) {
5162		PMD_INIT_LOG(ERR, "setting VF receive mode set should be done"
5163			     " on 82599 hardware and newer");
5164		return -ENOTSUP;
5165	}
5166	if (ixgbe_vt_check(hw) < 0)
5167		return -ENOTSUP;
5168
5169	val = ixgbe_convert_vm_rx_mask_to_val(rx_mask, val);
5170
5171	if (on)
5172		vmolr |= val;
5173	else
5174		vmolr &= ~val;
5175
5176	IXGBE_WRITE_REG(hw, IXGBE_VMOLR(vf), vmolr);
5177
5178	return 0;
5179}
5180
5181int
5182rte_pmd_ixgbe_set_vf_rx(uint8_t port, uint16_t vf, uint8_t on)
5183{
5184	struct rte_eth_dev *dev;
5185	struct rte_pci_device *pci_dev;
5186	uint32_t reg, addr;
5187	uint32_t val;
5188	const uint8_t bit1 = 0x1;
5189	struct ixgbe_hw *hw;
5190
5191	RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV);
5192
5193	dev = &rte_eth_devices[port];
5194	pci_dev = IXGBE_DEV_TO_PCI(dev);
5195
5196	if (!is_device_supported(dev, &rte_ixgbe_pmd))
5197		return -ENOTSUP;
5198
5199	if (vf >= pci_dev->max_vfs)
5200		return -EINVAL;
5201
5202	if (on > 1)
5203		return -EINVAL;
5204
5205	hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5206
5207	if (ixgbe_vt_check(hw) < 0)
5208		return -ENOTSUP;
5209
5210	/* for vf >= 32, set bit in PFVFRE[1], otherwise PFVFRE[0] */
5211	if (vf >= 32) {
5212		addr = IXGBE_VFRE(1);
5213		val = bit1 << (vf - 32);
5214	} else {
5215		addr = IXGBE_VFRE(0);
5216		val = bit1 << vf;
5217	}
5218
5219	reg = IXGBE_READ_REG(hw, addr);
5220
5221	if (on)
5222		reg |= val;
5223	else
5224		reg &= ~val;
5225
5226	IXGBE_WRITE_REG(hw, addr, reg);
5227
5228	return 0;
5229}
5230
5231int
5232rte_pmd_ixgbe_set_vf_tx(uint8_t port, uint16_t vf, uint8_t on)
5233{
5234	struct rte_eth_dev *dev;
5235	struct rte_pci_device *pci_dev;
5236	uint32_t reg, addr;
5237	uint32_t val;
5238	const uint8_t bit1 = 0x1;
5239
5240	struct ixgbe_hw *hw;
5241
5242	RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV);
5243
5244	dev = &rte_eth_devices[port];
5245	pci_dev = IXGBE_DEV_TO_PCI(dev);
5246
5247	if (!is_device_supported(dev, &rte_ixgbe_pmd))
5248		return -ENOTSUP;
5249
5250	if (vf >= pci_dev->max_vfs)
5251		return -EINVAL;
5252
5253	if (on > 1)
5254		return -EINVAL;
5255
5256	hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5257	if (ixgbe_vt_check(hw) < 0)
5258		return -ENOTSUP;
5259
5260	/* for vf >= 32, set bit in PFVFTE[1], otherwise PFVFTE[0] */
5261	if (vf >= 32) {
5262		addr = IXGBE_VFTE(1);
5263		val = bit1 << (vf - 32);
5264	} else {
5265		addr = IXGBE_VFTE(0);
5266		val = bit1 << vf;
5267	}
5268
5269	reg = IXGBE_READ_REG(hw, addr);
5270
5271	if (on)
5272		reg |= val;
5273	else
5274		reg &= ~val;
5275
5276	IXGBE_WRITE_REG(hw, addr, reg);
5277
5278	return 0;
5279}
5280
5281int
5282rte_pmd_ixgbe_set_vf_vlan_filter(uint8_t port, uint16_t vlan,
5283			uint64_t vf_mask, uint8_t vlan_on)
5284{
5285	struct rte_eth_dev *dev;
5286	int ret = 0;
5287	uint16_t vf_idx;
5288	struct ixgbe_hw *hw;
5289
5290	RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV);
5291
5292	dev = &rte_eth_devices[port];
5293
5294	if (!is_device_supported(dev, &rte_ixgbe_pmd))
5295		return -ENOTSUP;
5296
5297	if ((vlan > ETHER_MAX_VLAN_ID) || (vf_mask == 0))
5298		return -EINVAL;
5299
5300	hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5301	if (ixgbe_vt_check(hw) < 0)
5302		return -ENOTSUP;
5303
5304	for (vf_idx = 0; vf_idx < 64; vf_idx++) {
5305		if (vf_mask & ((uint64_t)(1ULL << vf_idx))) {
5306			ret = hw->mac.ops.set_vfta(hw, vlan, vf_idx,
5307						   vlan_on, false);
5308			if (ret < 0)
5309				return ret;
5310		}
5311	}
5312
5313	return ret;
5314}
5315
5316int rte_pmd_ixgbe_set_vf_rate_limit(uint8_t port, uint16_t vf,
5317	uint16_t tx_rate, uint64_t q_msk)
5318{
5319	struct rte_eth_dev *dev;
5320	struct ixgbe_hw *hw;
5321	struct ixgbe_vf_info *vfinfo;
5322	struct rte_eth_link link;
5323	uint8_t  nb_q_per_pool;
5324	uint32_t queue_stride;
5325	uint32_t queue_idx, idx = 0, vf_idx;
5326	uint32_t queue_end;
5327	uint16_t total_rate = 0;
5328	struct rte_pci_device *pci_dev;
5329
5330	RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV);
5331
5332	dev = &rte_eth_devices[port];
5333	pci_dev = IXGBE_DEV_TO_PCI(dev);
5334	rte_eth_link_get_nowait(port, &link);
5335
5336	if (!is_device_supported(dev, &rte_ixgbe_pmd))
5337		return -ENOTSUP;
5338
5339	if (vf >= pci_dev->max_vfs)
5340		return -EINVAL;
5341
5342	if (tx_rate > link.link_speed)
5343		return -EINVAL;
5344
5345	if (q_msk == 0)
5346		return 0;
5347
5348	hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5349	vfinfo = *(IXGBE_DEV_PRIVATE_TO_P_VFDATA(dev->data->dev_private));
5350	nb_q_per_pool = RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool;
5351	queue_stride = IXGBE_MAX_RX_QUEUE_NUM / RTE_ETH_DEV_SRIOV(dev).active;
5352	queue_idx = vf * queue_stride;
5353	queue_end = queue_idx + nb_q_per_pool - 1;
5354	if (queue_end >= hw->mac.max_tx_queues)
5355		return -EINVAL;
5356
5357	if (vfinfo) {
5358		for (vf_idx = 0; vf_idx < pci_dev->max_vfs; vf_idx++) {
5359			if (vf_idx == vf)
5360				continue;
5361			for (idx = 0; idx < RTE_DIM(vfinfo[vf_idx].tx_rate);
5362				idx++)
5363				total_rate += vfinfo[vf_idx].tx_rate[idx];
5364		}
5365	} else {
5366		return -EINVAL;
5367	}
5368
5369	/* Store tx_rate for this vf. */
5370	for (idx = 0; idx < nb_q_per_pool; idx++) {
5371		if (((uint64_t)0x1 << idx) & q_msk) {
5372			if (vfinfo[vf].tx_rate[idx] != tx_rate)
5373				vfinfo[vf].tx_rate[idx] = tx_rate;
5374			total_rate += tx_rate;
5375		}
5376	}
5377
5378	if (total_rate > dev->data->dev_link.link_speed) {
5379		/* Reset stored TX rate of the VF if it causes exceed
5380		 * link speed.
5381		 */
5382		memset(vfinfo[vf].tx_rate, 0, sizeof(vfinfo[vf].tx_rate));
5383		return -EINVAL;
5384	}
5385
5386	/* Set RTTBCNRC of each queue/pool for vf X  */
5387	for (; queue_idx <= queue_end; queue_idx++) {
5388		if (0x1 & q_msk)
5389			ixgbe_set_queue_rate_limit(dev, queue_idx, tx_rate);
5390		q_msk = q_msk >> 1;
5391	}
5392
5393	return 0;
5394}
5395
5396#define IXGBE_MRCTL_VPME  0x01 /* Virtual Pool Mirroring. */
5397#define IXGBE_MRCTL_UPME  0x02 /* Uplink Port Mirroring. */
5398#define IXGBE_MRCTL_DPME  0x04 /* Downlink Port Mirroring. */
5399#define IXGBE_MRCTL_VLME  0x08 /* VLAN Mirroring. */
5400#define IXGBE_INVALID_MIRROR_TYPE(mirror_type) \
5401	((mirror_type) & ~(uint8_t)(ETH_MIRROR_VIRTUAL_POOL_UP | \
5402	ETH_MIRROR_UPLINK_PORT | ETH_MIRROR_DOWNLINK_PORT | ETH_MIRROR_VLAN))
5403
5404static int
5405ixgbe_mirror_rule_set(struct rte_eth_dev *dev,
5406			struct rte_eth_mirror_conf *mirror_conf,
5407			uint8_t rule_id, uint8_t on)
5408{
5409	uint32_t mr_ctl, vlvf;
5410	uint32_t mp_lsb = 0;
5411	uint32_t mv_msb = 0;
5412	uint32_t mv_lsb = 0;
5413	uint32_t mp_msb = 0;
5414	uint8_t i = 0;
5415	int reg_index = 0;
5416	uint64_t vlan_mask = 0;
5417
5418	const uint8_t pool_mask_offset = 32;
5419	const uint8_t vlan_mask_offset = 32;
5420	const uint8_t dst_pool_offset = 8;
5421	const uint8_t rule_mr_offset  = 4;
5422	const uint8_t mirror_rule_mask = 0x0F;
5423
5424	struct ixgbe_mirror_info *mr_info =
5425			(IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private));
5426	struct ixgbe_hw *hw =
5427		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5428	uint8_t mirror_type = 0;
5429
5430	if (ixgbe_vt_check(hw) < 0)
5431		return -ENOTSUP;
5432
5433	if (rule_id >= IXGBE_MAX_MIRROR_RULES)
5434		return -EINVAL;
5435
5436	if (IXGBE_INVALID_MIRROR_TYPE(mirror_conf->rule_type)) {
5437		PMD_DRV_LOG(ERR, "unsupported mirror type 0x%x.",
5438			mirror_conf->rule_type);
5439		return -EINVAL;
5440	}
5441
5442	if (mirror_conf->rule_type & ETH_MIRROR_VLAN) {
5443		mirror_type |= IXGBE_MRCTL_VLME;
5444		/* Check if vlan id is valid and find conresponding VLAN ID index in VLVF */
5445		for (i = 0; i < IXGBE_VLVF_ENTRIES; i++) {
5446			if (mirror_conf->vlan.vlan_mask & (1ULL << i)) {
5447				/* search vlan id related pool vlan filter index */
5448				reg_index = ixgbe_find_vlvf_slot(hw,
5449						 mirror_conf->vlan.vlan_id[i],
5450						 false);
5451				if (reg_index < 0)
5452					return -EINVAL;
5453				vlvf = IXGBE_READ_REG(hw, IXGBE_VLVF(reg_index));
5454				if ((vlvf & IXGBE_VLVF_VIEN) &&
5455				    ((vlvf & IXGBE_VLVF_VLANID_MASK) ==
5456				      mirror_conf->vlan.vlan_id[i]))
5457					vlan_mask |= (1ULL << reg_index);
5458				else
5459					return -EINVAL;
5460			}
5461		}
5462
5463		if (on) {
5464			mv_lsb = vlan_mask & 0xFFFFFFFF;
5465			mv_msb = vlan_mask >> vlan_mask_offset;
5466
5467			mr_info->mr_conf[rule_id].vlan.vlan_mask =
5468						mirror_conf->vlan.vlan_mask;
5469			for (i = 0; i < ETH_VMDQ_MAX_VLAN_FILTERS; i++) {
5470				if (mirror_conf->vlan.vlan_mask & (1ULL << i))
5471					mr_info->mr_conf[rule_id].vlan.vlan_id[i] =
5472						mirror_conf->vlan.vlan_id[i];
5473			}
5474		} else {
5475			mv_lsb = 0;
5476			mv_msb = 0;
5477			mr_info->mr_conf[rule_id].vlan.vlan_mask = 0;
5478			for (i = 0; i < ETH_VMDQ_MAX_VLAN_FILTERS; i++)
5479				mr_info->mr_conf[rule_id].vlan.vlan_id[i] = 0;
5480		}
5481	}
5482
5483	/*
5484	 * if enable pool mirror, write related pool mask register,if disable
5485	 * pool mirror, clear PFMRVM register
5486	 */
5487	if (mirror_conf->rule_type & ETH_MIRROR_VIRTUAL_POOL_UP) {
5488		mirror_type |= IXGBE_MRCTL_VPME;
5489		if (on) {
5490			mp_lsb = mirror_conf->pool_mask & 0xFFFFFFFF;
5491			mp_msb = mirror_conf->pool_mask >> pool_mask_offset;
5492			mr_info->mr_conf[rule_id].pool_mask =
5493					mirror_conf->pool_mask;
5494
5495		} else {
5496			mp_lsb = 0;
5497			mp_msb = 0;
5498			mr_info->mr_conf[rule_id].pool_mask = 0;
5499		}
5500	}
5501	if (mirror_conf->rule_type & ETH_MIRROR_UPLINK_PORT)
5502		mirror_type |= IXGBE_MRCTL_UPME;
5503	if (mirror_conf->rule_type & ETH_MIRROR_DOWNLINK_PORT)
5504		mirror_type |= IXGBE_MRCTL_DPME;
5505
5506	/* read  mirror control register and recalculate it */
5507	mr_ctl = IXGBE_READ_REG(hw, IXGBE_MRCTL(rule_id));
5508
5509	if (on) {
5510		mr_ctl |= mirror_type;
5511		mr_ctl &= mirror_rule_mask;
5512		mr_ctl |= mirror_conf->dst_pool << dst_pool_offset;
5513	} else
5514		mr_ctl &= ~(mirror_conf->rule_type & mirror_rule_mask);
5515
5516	mr_info->mr_conf[rule_id].rule_type = mirror_conf->rule_type;
5517	mr_info->mr_conf[rule_id].dst_pool = mirror_conf->dst_pool;
5518
5519	/* write mirrror control  register */
5520	IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl);
5521
5522	/* write pool mirrror control  register */
5523	if (mirror_conf->rule_type == ETH_MIRROR_VIRTUAL_POOL_UP) {
5524		IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), mp_lsb);
5525		IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset),
5526				mp_msb);
5527	}
5528	/* write VLAN mirrror control  register */
5529	if (mirror_conf->rule_type == ETH_MIRROR_VLAN) {
5530		IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), mv_lsb);
5531		IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset),
5532				mv_msb);
5533	}
5534
5535	return 0;
5536}
5537
5538static int
5539ixgbe_mirror_rule_reset(struct rte_eth_dev *dev, uint8_t rule_id)
5540{
5541	int mr_ctl = 0;
5542	uint32_t lsb_val = 0;
5543	uint32_t msb_val = 0;
5544	const uint8_t rule_mr_offset = 4;
5545
5546	struct ixgbe_hw *hw =
5547		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5548	struct ixgbe_mirror_info *mr_info =
5549		(IXGBE_DEV_PRIVATE_TO_PFDATA(dev->data->dev_private));
5550
5551	if (ixgbe_vt_check(hw) < 0)
5552		return -ENOTSUP;
5553
5554	memset(&mr_info->mr_conf[rule_id], 0,
5555		sizeof(struct rte_eth_mirror_conf));
5556
5557	/* clear PFVMCTL register */
5558	IXGBE_WRITE_REG(hw, IXGBE_MRCTL(rule_id), mr_ctl);
5559
5560	/* clear pool mask register */
5561	IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id), lsb_val);
5562	IXGBE_WRITE_REG(hw, IXGBE_VMRVM(rule_id + rule_mr_offset), msb_val);
5563
5564	/* clear vlan mask register */
5565	IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id), lsb_val);
5566	IXGBE_WRITE_REG(hw, IXGBE_VMRVLAN(rule_id + rule_mr_offset), msb_val);
5567
5568	return 0;
5569}
5570
5571static int
5572ixgbevf_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
5573{
5574	struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(dev);
5575	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
5576	uint32_t mask;
5577	struct ixgbe_hw *hw =
5578		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5579
5580	mask = IXGBE_READ_REG(hw, IXGBE_VTEIMS);
5581	mask |= (1 << IXGBE_MISC_VEC_ID);
5582	RTE_SET_USED(queue_id);
5583	IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
5584
5585	rte_intr_enable(intr_handle);
5586
5587	return 0;
5588}
5589
5590static int
5591ixgbevf_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
5592{
5593	uint32_t mask;
5594	struct ixgbe_hw *hw =
5595		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5596
5597	mask = IXGBE_READ_REG(hw, IXGBE_VTEIMS);
5598	mask &= ~(1 << IXGBE_MISC_VEC_ID);
5599	RTE_SET_USED(queue_id);
5600	IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, mask);
5601
5602	return 0;
5603}
5604
5605static int
5606ixgbe_dev_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
5607{
5608	struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(dev);
5609	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
5610	uint32_t mask;
5611	struct ixgbe_hw *hw =
5612		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5613	struct ixgbe_interrupt *intr =
5614		IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
5615
5616	if (queue_id < 16) {
5617		ixgbe_disable_intr(hw);
5618		intr->mask |= (1 << queue_id);
5619		ixgbe_enable_intr(dev);
5620	} else if (queue_id < 32) {
5621		mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(0));
5622		mask &= (1 << queue_id);
5623		IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
5624	} else if (queue_id < 64) {
5625		mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(1));
5626		mask &= (1 << (queue_id - 32));
5627		IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
5628	}
5629	rte_intr_enable(intr_handle);
5630
5631	return 0;
5632}
5633
5634static int
5635ixgbe_dev_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
5636{
5637	uint32_t mask;
5638	struct ixgbe_hw *hw =
5639		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5640	struct ixgbe_interrupt *intr =
5641		IXGBE_DEV_PRIVATE_TO_INTR(dev->data->dev_private);
5642
5643	if (queue_id < 16) {
5644		ixgbe_disable_intr(hw);
5645		intr->mask &= ~(1 << queue_id);
5646		ixgbe_enable_intr(dev);
5647	} else if (queue_id < 32) {
5648		mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(0));
5649		mask &= ~(1 << queue_id);
5650		IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
5651	} else if (queue_id < 64) {
5652		mask = IXGBE_READ_REG(hw, IXGBE_EIMS_EX(1));
5653		mask &= ~(1 << (queue_id - 32));
5654		IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
5655	}
5656
5657	return 0;
5658}
5659
5660static void
5661ixgbevf_set_ivar_map(struct ixgbe_hw *hw, int8_t direction,
5662		     uint8_t queue, uint8_t msix_vector)
5663{
5664	uint32_t tmp, idx;
5665
5666	if (direction == -1) {
5667		/* other causes */
5668		msix_vector |= IXGBE_IVAR_ALLOC_VAL;
5669		tmp = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
5670		tmp &= ~0xFF;
5671		tmp |= msix_vector;
5672		IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, tmp);
5673	} else {
5674		/* rx or tx cause */
5675		msix_vector |= IXGBE_IVAR_ALLOC_VAL;
5676		idx = ((16 * (queue & 1)) + (8 * direction));
5677		tmp = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1));
5678		tmp &= ~(0xFF << idx);
5679		tmp |= (msix_vector << idx);
5680		IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), tmp);
5681	}
5682}
5683
5684/**
5685 * set the IVAR registers, mapping interrupt causes to vectors
5686 * @param hw
5687 *  pointer to ixgbe_hw struct
5688 * @direction
5689 *  0 for Rx, 1 for Tx, -1 for other causes
5690 * @queue
5691 *  queue to map the corresponding interrupt to
5692 * @msix_vector
5693 *  the vector to map to the corresponding queue
5694 */
5695static void
5696ixgbe_set_ivar_map(struct ixgbe_hw *hw, int8_t direction,
5697		   uint8_t queue, uint8_t msix_vector)
5698{
5699	uint32_t tmp, idx;
5700
5701	msix_vector |= IXGBE_IVAR_ALLOC_VAL;
5702	if (hw->mac.type == ixgbe_mac_82598EB) {
5703		if (direction == -1)
5704			direction = 0;
5705		idx = (((direction * 64) + queue) >> 2) & 0x1F;
5706		tmp = IXGBE_READ_REG(hw, IXGBE_IVAR(idx));
5707		tmp &= ~(0xFF << (8 * (queue & 0x3)));
5708		tmp |= (msix_vector << (8 * (queue & 0x3)));
5709		IXGBE_WRITE_REG(hw, IXGBE_IVAR(idx), tmp);
5710	} else if ((hw->mac.type == ixgbe_mac_82599EB) ||
5711			(hw->mac.type == ixgbe_mac_X540)) {
5712		if (direction == -1) {
5713			/* other causes */
5714			idx = ((queue & 1) * 8);
5715			tmp = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC);
5716			tmp &= ~(0xFF << idx);
5717			tmp |= (msix_vector << idx);
5718			IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, tmp);
5719		} else {
5720			/* rx or tx causes */
5721			idx = ((16 * (queue & 1)) + (8 * direction));
5722			tmp = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1));
5723			tmp &= ~(0xFF << idx);
5724			tmp |= (msix_vector << idx);
5725			IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), tmp);
5726		}
5727	}
5728}
5729
5730static void
5731ixgbevf_configure_msix(struct rte_eth_dev *dev)
5732{
5733	struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(dev);
5734	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
5735	struct ixgbe_hw *hw =
5736		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5737	uint32_t q_idx;
5738	uint32_t vector_idx = IXGBE_MISC_VEC_ID;
5739
5740	/* Configure VF other cause ivar */
5741	ixgbevf_set_ivar_map(hw, -1, 1, vector_idx);
5742
5743	/* won't configure msix register if no mapping is done
5744	 * between intr vector and event fd.
5745	 */
5746	if (!rte_intr_dp_is_en(intr_handle))
5747		return;
5748
5749	/* Configure all RX queues of VF */
5750	for (q_idx = 0; q_idx < dev->data->nb_rx_queues; q_idx++) {
5751		/* Force all queue use vector 0,
5752		 * as IXGBE_VF_MAXMSIVECOTR = 1
5753		 */
5754		ixgbevf_set_ivar_map(hw, 0, q_idx, vector_idx);
5755		intr_handle->intr_vec[q_idx] = vector_idx;
5756	}
5757}
5758
5759/**
5760 * Sets up the hardware to properly generate MSI-X interrupts
5761 * @hw
5762 *  board private structure
5763 */
5764static void
5765ixgbe_configure_msix(struct rte_eth_dev *dev)
5766{
5767	struct rte_pci_device *pci_dev = IXGBE_DEV_TO_PCI(dev);
5768	struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
5769	struct ixgbe_hw *hw =
5770		IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5771	uint32_t queue_id, base = IXGBE_MISC_VEC_ID;
5772	uint32_t vec = IXGBE_MISC_VEC_ID;
5773	uint32_t mask;
5774	uint32_t gpie;
5775
5776	/* won't configure msix register if no mapping is done
5777	 * between intr vector and event fd
5778	 */
5779	if (!rte_intr_dp_is_en(intr_handle))
5780		return;
5781
5782	if (rte_intr_allow_others(intr_handle))
5783		vec = base = IXGBE_RX_VEC_START;
5784
5785	/* setup GPIE for MSI-x mode */
5786	gpie = IXGBE_READ_REG(hw, IXGBE_GPIE);
5787	gpie |= IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT |
5788		IXGBE_GPIE_OCD | IXGBE_GPIE_EIAME;
5789	/* auto clearing and auto setting corresponding bits in EIMS
5790	 * when MSI-X interrupt is triggered
5791	 */
5792	if (hw->mac.type == ixgbe_mac_82598EB) {
5793		IXGBE_WRITE_REG(hw, IXGBE_EIAM, IXGBE_EICS_RTX_QUEUE);
5794	} else {
5795		IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(0), 0xFFFFFFFF);
5796		IXGBE_WRITE_REG(hw, IXGBE_EIAM_EX(1), 0xFFFFFFFF);
5797	}
5798	IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie);
5799
5800	/* Populate the IVAR table and set the ITR values to the
5801	 * corresponding register.
5802	 */
5803	for (queue_id = 0; queue_id < dev->data->nb_rx_queues;
5804	     queue_id++) {
5805		/* by default, 1:1 mapping */
5806		ixgbe_set_ivar_map(hw, 0, queue_id, vec);
5807		intr_handle->intr_vec[queue_id] = vec;
5808		if (vec < base + intr_handle->nb_efd - 1)
5809			vec++;
5810	}
5811
5812	switch (hw->mac.type) {
5813	case ixgbe_mac_82598EB:
5814		ixgbe_set_ivar_map(hw, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX,
5815				   IXGBE_MISC_VEC_ID);
5816		break;
5817	case ixgbe_mac_82599EB:
5818	case ixgbe_mac_X540:
5819		ixgbe_set_ivar_map(hw, -1, 1, IXGBE_MISC_VEC_ID);
5820		break;
5821	default:
5822		break;
5823	}
5824	IXGBE_WRITE_REG(hw, IXGBE_EITR(IXGBE_MISC_VEC_ID),
5825			IXGBE_MIN_INTER_INTERRUPT_INTERVAL_DEFAULT & 0xFFF);
5826
5827	/* set up to autoclear timer, and the vectors */
5828	mask = IXGBE_EIMS_ENABLE_MASK;
5829	mask &= ~(IXGBE_EIMS_OTHER |
5830		  IXGBE_EIMS_MAILBOX |
5831		  IXGBE_EIMS_LSC);
5832
5833	IXGBE_WRITE_REG(hw, IXGBE_EIAC, mask);
5834}
5835
5836static int ixgbe_set_queue_rate_limit(struct rte_eth_dev *dev,
5837	uint16_t queue_idx, uint16_t tx_rate)
5838{
5839	struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5840	uint32_t rf_dec, rf_int;
5841	uint32_t bcnrc_val;
5842	uint16_t link_speed = dev->data->dev_link.link_speed;
5843
5844	if (queue_idx >= hw->mac.max_tx_queues)
5845		return -EINVAL;
5846
5847	if (tx_rate != 0) {
5848		/* Calculate the rate factor values to set */
5849		rf_int = (uint32_t)link_speed / (uint32_t)tx_rate;
5850		rf_dec = (uint32_t)link_speed % (uint32_t)tx_rate;
5851		rf_dec = (rf_dec << IXGBE_RTTBCNRC_RF_INT_SHIFT) / tx_rate;
5852
5853		bcnrc_val = IXGBE_RTTBCNRC_RS_ENA;
5854		bcnrc_val |= ((rf_int << IXGBE_RTTBCNRC_RF_INT_SHIFT) &
5855				IXGBE_RTTBCNRC_RF_INT_MASK_M);
5856		bcnrc_val |= (rf_dec & IXGBE_RTTBCNRC_RF_DEC_MASK);
5857	} else {
5858		bcnrc_val = 0;
5859	}
5860
5861	/*
5862	 * Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
5863	 * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported, otherwise
5864	 * set as 0x4.
5865	 */
5866	if ((dev->data->dev_conf.rxmode.jumbo_frame == 1) &&
5867		(dev->data->dev_conf.rxmode.max_rx_pkt_len >=
5868				IXGBE_MAX_JUMBO_FRAME_SIZE))
5869		IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRM,
5870			IXGBE_MMW_SIZE_JUMBO_FRAME);
5871	else
5872		IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRM,
5873			IXGBE_MMW_SIZE_DEFAULT);
5874
5875	/* Set RTTBCNRC of queue X */
5876	IXGBE_WRITE_REG(hw, IXGBE_RTTDQSEL, queue_idx);
5877	IXGBE_WRITE_REG(hw, IXGBE_RTTBCNRC, bcnrc_val);
5878	IXGBE_WRITE_FLUSH(hw);
5879
5880	return 0;
5881}
5882
5883static void
5884ixgbevf_add_mac_addr(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
5885		     __attribute__((unused)) uint32_t index,
5886		     __attribute__((unused)) uint32_t pool)
5887{
5888	struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5889	int diag;
5890
5891	/*
5892	 * On a 82599 VF, adding again the same MAC addr is not an idempotent
5893	 * operation. Trap this case to avoid exhausting the [very limited]
5894	 * set of PF resources used to store VF MAC addresses.
5895	 */
5896	if (memcmp(hw->mac.perm_addr, mac_addr, sizeof(struct ether_addr)) == 0)
5897		return;
5898	diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes);
5899	if (diag == 0)
5900		return;
5901	PMD_DRV_LOG(ERR, "Unable to add MAC address - diag=%d", diag);
5902}
5903
5904static void
5905ixgbevf_remove_mac_addr(struct rte_eth_dev *dev, uint32_t index)
5906{
5907	struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5908	struct ether_addr *perm_addr = (struct ether_addr *) hw->mac.perm_addr;
5909	struct ether_addr *mac_addr;
5910	uint32_t i;
5911	int diag;
5912
5913	/*
5914	 * The IXGBE_VF_SET_MACVLAN command of the ixgbe-pf driver does
5915	 * not support the deletion of a given MAC address.
5916	 * Instead, it imposes to delete all MAC addresses, then to add again
5917	 * all MAC addresses with the exception of the one to be deleted.
5918	 */
5919	(void) ixgbevf_set_uc_addr_vf(hw, 0, NULL);
5920
5921	/*
5922	 * Add again all MAC addresses, with the exception of the deleted one
5923	 * and of the permanent MAC address.
5924	 */
5925	for (i = 0, mac_addr = dev->data->mac_addrs;
5926	     i < hw->mac.num_rar_entries; i++, mac_addr++) {
5927		/* Skip the deleted MAC address */
5928		if (i == index)
5929			continue;
5930		/* Skip NULL MAC addresses */
5931		if (is_zero_ether_addr(mac_addr))
5932			continue;
5933		/* Skip the permanent MAC address */
5934		if (memcmp(perm_addr, mac_addr, sizeof(struct ether_addr)) == 0)
5935			continue;
5936		diag = ixgbevf_set_uc_addr_vf(hw, 2, mac_addr->addr_bytes);
5937		if (diag != 0)
5938			PMD_DRV_LOG(ERR,
5939				    "Adding again MAC address "
5940				    "%02x:%02x:%02x:%02x:%02x:%02x failed "
5941				    "diag=%d",
5942				    mac_addr->addr_bytes[0],
5943				    mac_addr->addr_bytes[1],
5944				    mac_addr->addr_bytes[2],
5945				    mac_addr->addr_bytes[3],
5946				    mac_addr->addr_bytes[4],
5947				    mac_addr->addr_bytes[5],
5948				    diag);
5949	}
5950}
5951
5952static void
5953ixgbevf_set_default_mac_addr(struct rte_eth_dev *dev, struct ether_addr *addr)
5954{
5955	struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5956
5957	hw->mac.ops.set_rar(hw, 0, (void *)addr, 0, 0);
5958}
5959
5960#define MAC_TYPE_FILTER_SUP(type)    do {\
5961	if ((type) != ixgbe_mac_82599EB && (type) != ixgbe_mac_X540 &&\
5962		(type) != ixgbe_mac_X550 && (type) != ixgbe_mac_X550EM_x &&\
5963		(type) != ixgbe_mac_X550EM_a)\
5964		return -ENOTSUP;\
5965} while (0)
5966
5967int
5968ixgbe_syn_filter_set(struct rte_eth_dev *dev,
5969			struct rte_eth_syn_filter *filter,
5970			bool add)
5971{
5972	struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
5973	struct ixgbe_filter_info *filter_info =
5974		IXGBE_DEV_PRIVATE_TO_FILTER_INFO(dev->data->dev_private);
5975	uint32_t syn_info;
5976	uint32_t synqf;
5977
5978	if (filter->queue >= IXGBE_MAX_RX_QUEUE_NUM)
5979		return -EINVAL;
5980
5981	syn_info = filter_info->syn_info;
5982
5983	if (add) {
5984		if (syn_info & IXGBE_SYN_FILTER_ENABLE)
5985			return -EINVAL;
5986		synqf = (uint32_t)(((filter->queue << IXGBE_SYN_FILTER_QUEUE_SHIFT) &
5987			IXGBE_SYN_FILTER_QUEUE) | IXGBE_SYN_FILTER_ENABLE);
5988
5989		if (filter->hig_pri)
5990			synqf |= IXGBE_SYN_FILTER_SYNQFP;
5991		else
5992			synqf &= ~IXGBE_SYN_FILTER_SYNQFP;
5993	} else {
5994		synqf = IXGBE_READ_REG(hw, IXGBE_SYNQF);
5995		if (!(syn_info & IXGBE_SYN_FILTER_ENABLE))
5996			return -ENOENT;
5997		synqf &= ~(IXGBE_SYN_FILTER_QUEUE | IXGBE_SYN_FILTER_ENABLE);
5998	}
5999
6000	filter_info->syn_info = synqf;
6001	IXGBE_WRITE_REG(hw,