18b52a31eSHanoh Haim/*-
28b52a31eSHanoh Haim * GPL LICENSE SUMMARY
38b52a31eSHanoh Haim *
48b52a31eSHanoh Haim *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
58b52a31eSHanoh Haim *
68b52a31eSHanoh Haim *   This program is free software; you can redistribute it and/or modify
78b52a31eSHanoh Haim *   it under the terms of version 2 of the GNU General Public License as
88b52a31eSHanoh Haim *   published by the Free Software Foundation.
98b52a31eSHanoh Haim *
108b52a31eSHanoh Haim *   This program is distributed in the hope that it will be useful, but
118b52a31eSHanoh Haim *   WITHOUT ANY WARRANTY; without even the implied warranty of
128b52a31eSHanoh Haim *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
138b52a31eSHanoh Haim *   General Public License for more details.
148b52a31eSHanoh Haim *
158b52a31eSHanoh Haim *   You should have received a copy of the GNU General Public License
168b52a31eSHanoh Haim *   along with this program; if not, write to the Free Software
178b52a31eSHanoh Haim *   Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
188b52a31eSHanoh Haim *   The full GNU General Public License is included in this distribution
198b52a31eSHanoh Haim *   in the file called LICENSE.GPL.
208b52a31eSHanoh Haim *
218b52a31eSHanoh Haim *   Contact Information:
228b52a31eSHanoh Haim *   Intel Corporation
238b52a31eSHanoh Haim */
248b52a31eSHanoh Haim
258b52a31eSHanoh Haim#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
268b52a31eSHanoh Haim
278b52a31eSHanoh Haim#include <linux/device.h>
288b52a31eSHanoh Haim#include <linux/module.h>
298b52a31eSHanoh Haim#include <linux/pci.h>
308b52a31eSHanoh Haim#include <linux/uio_driver.h>
318b52a31eSHanoh Haim#include <linux/io.h>
328b52a31eSHanoh Haim#include <linux/msi.h>
338b52a31eSHanoh Haim#include <linux/version.h>
340400a950Simarom#include <linux/slab.h>
358b52a31eSHanoh Haim
368b52a31eSHanoh Haim#ifdef CONFIG_XEN_DOM0
378b52a31eSHanoh Haim#include <xen/xen.h>
388b52a31eSHanoh Haim#endif
398b52a31eSHanoh Haim#include "rte_pci_dev_features.h"
408b52a31eSHanoh Haim
418b52a31eSHanoh Haim#include "compat.h"
428b52a31eSHanoh Haim
438b52a31eSHanoh Haim/**
448b52a31eSHanoh Haim * A structure describing the private information for a uio device.
458b52a31eSHanoh Haim */
468b52a31eSHanoh Haimstruct rte_uio_pci_dev {
478b52a31eSHanoh Haim	struct uio_info info;
488b52a31eSHanoh Haim	struct pci_dev *pdev;
498b52a31eSHanoh Haim	enum rte_intr_mode mode;
508b52a31eSHanoh Haim};
518b52a31eSHanoh Haim
520400a950Simaromstatic char *intr_mode;
538b52a31eSHanoh Haimstatic enum rte_intr_mode igbuio_intr_mode_preferred = RTE_INTR_MODE_MSIX;
548b52a31eSHanoh Haim
558b52a31eSHanoh Haim/* sriov sysfs */
568b52a31eSHanoh Haimstatic ssize_t
578b52a31eSHanoh Haimshow_max_vfs(struct device *dev, struct device_attribute *attr,
588b52a31eSHanoh Haim	     char *buf)
598b52a31eSHanoh Haim{
600400a950Simarom	return snprintf(buf, 10, "%u\n", dev_num_vf(dev));
618b52a31eSHanoh Haim}
628b52a31eSHanoh Haim
638b52a31eSHanoh Haimstatic ssize_t
648b52a31eSHanoh Haimstore_max_vfs(struct device *dev, struct device_attribute *attr,
658b52a31eSHanoh Haim	      const char *buf, size_t count)
668b52a31eSHanoh Haim{
678b52a31eSHanoh Haim	int err = 0;
688b52a31eSHanoh Haim	unsigned long max_vfs;
690400a950Simarom	struct pci_dev *pdev = to_pci_dev(dev);
708b52a31eSHanoh Haim
718b52a31eSHanoh Haim	if (0 != kstrtoul(buf, 0, &max_vfs))
728b52a31eSHanoh Haim		return -EINVAL;
738b52a31eSHanoh Haim
748b52a31eSHanoh Haim	if (0 == max_vfs)
758b52a31eSHanoh Haim		pci_disable_sriov(pdev);
768b52a31eSHanoh Haim	else if (0 == pci_num_vf(pdev))
778b52a31eSHanoh Haim		err = pci_enable_sriov(pdev, max_vfs);
788b52a31eSHanoh Haim	else /* do nothing if change max_vfs number */
798b52a31eSHanoh Haim		err = -EINVAL;
808b52a31eSHanoh Haim
818b52a31eSHanoh Haim	return err ? err : count;
828b52a31eSHanoh Haim}
838b52a31eSHanoh Haim
848b52a31eSHanoh Haim#ifdef RTE_PCI_CONFIG
858b52a31eSHanoh Haimstatic ssize_t
868b52a31eSHanoh Haimshow_extended_tag(struct device *dev, struct device_attribute *attr, char *buf)
878b52a31eSHanoh Haim{
880400a950Simarom	dev_info(dev, "Deprecated\n");
898b52a31eSHanoh Haim
900400a950Simarom	return 0;
918b52a31eSHanoh Haim}
928b52a31eSHanoh Haim
938b52a31eSHanoh Haimstatic ssize_t
948b52a31eSHanoh Haimstore_extended_tag(struct device *dev,
958b52a31eSHanoh Haim		   struct device_attribute *attr,
968b52a31eSHanoh Haim		   const char *buf,
978b52a31eSHanoh Haim		   size_t count)
988b52a31eSHanoh Haim{
990400a950Simarom	dev_info(dev, "Deprecated\n");
1008b52a31eSHanoh Haim
1010400a950Simarom	return 0;
1028b52a31eSHanoh Haim}
1038b52a31eSHanoh Haim
1048b52a31eSHanoh Haimstatic ssize_t
1058b52a31eSHanoh Haimshow_max_read_request_size(struct device *dev,
1068b52a31eSHanoh Haim			   struct device_attribute *attr,
1078b52a31eSHanoh Haim			   char *buf)
1088b52a31eSHanoh Haim{
1090400a950Simarom	dev_info(dev, "Deprecated\n");
1108b52a31eSHanoh Haim
1110400a950Simarom	return 0;
1128b52a31eSHanoh Haim}
1138b52a31eSHanoh Haim
1148b52a31eSHanoh Haimstatic ssize_t
1158b52a31eSHanoh Haimstore_max_read_request_size(struct device *dev,
1168b52a31eSHanoh Haim			    struct device_attribute *attr,
1178b52a31eSHanoh Haim			    const char *buf,
1188b52a31eSHanoh Haim			    size_t count)
1198b52a31eSHanoh Haim{
1200400a950Simarom	dev_info(dev, "Deprecated\n");
1218b52a31eSHanoh Haim
1220400a950Simarom	return 0;
1238b52a31eSHanoh Haim}
1248b52a31eSHanoh Haim#endif
1258b52a31eSHanoh Haim
1268b52a31eSHanoh Haimstatic DEVICE_ATTR(max_vfs, S_IRUGO | S_IWUSR, show_max_vfs, store_max_vfs);
1278b52a31eSHanoh Haim#ifdef RTE_PCI_CONFIG
1288b52a31eSHanoh Haimstatic DEVICE_ATTR(extended_tag, S_IRUGO | S_IWUSR, show_extended_tag,
1298b52a31eSHanoh Haim	store_extended_tag);
1308b52a31eSHanoh Haimstatic DEVICE_ATTR(max_read_request_size, S_IRUGO | S_IWUSR,
1318b52a31eSHanoh Haim	show_max_read_request_size, store_max_read_request_size);
1328b52a31eSHanoh Haim#endif
1338b52a31eSHanoh Haim
1348b52a31eSHanoh Haimstatic struct attribute *dev_attrs[] = {
1358b52a31eSHanoh Haim	&dev_attr_max_vfs.attr,
1368b52a31eSHanoh Haim#ifdef RTE_PCI_CONFIG
1378b52a31eSHanoh Haim	&dev_attr_extended_tag.attr,
1388b52a31eSHanoh Haim	&dev_attr_max_read_request_size.attr,
1398b52a31eSHanoh Haim#endif
1408b52a31eSHanoh Haim	NULL,
1418b52a31eSHanoh Haim};
1428b52a31eSHanoh Haim
1438b52a31eSHanoh Haimstatic const struct attribute_group dev_attr_grp = {
1448b52a31eSHanoh Haim	.attrs = dev_attrs,
1458b52a31eSHanoh Haim};
1468b52a31eSHanoh Haim/*
1478b52a31eSHanoh Haim * It masks the msix on/off of generating MSI-X messages.
1488b52a31eSHanoh Haim */
1498b52a31eSHanoh Haimstatic void
1508b52a31eSHanoh Haimigbuio_msix_mask_irq(struct msi_desc *desc, int32_t state)
1518b52a31eSHanoh Haim{
1528b52a31eSHanoh Haim	u32 mask_bits = desc->masked;
1538b52a31eSHanoh Haim	unsigned offset = desc->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE +
1548b52a31eSHanoh Haim						PCI_MSIX_ENTRY_VECTOR_CTRL;
1558b52a31eSHanoh Haim
1568b52a31eSHanoh Haim	if (state != 0)
1578b52a31eSHanoh Haim		mask_bits &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT;
1588b52a31eSHanoh Haim	else
1598b52a31eSHanoh Haim		mask_bits |= PCI_MSIX_ENTRY_CTRL_MASKBIT;
1608b52a31eSHanoh Haim
1618b52a31eSHanoh Haim	if (mask_bits != desc->masked) {
1628b52a31eSHanoh Haim		writel(mask_bits, desc->mask_base + offset);
1638b52a31eSHanoh Haim		readl(desc->mask_base);
1648b52a31eSHanoh Haim		desc->masked = mask_bits;
1658b52a31eSHanoh Haim	}
1668b52a31eSHanoh Haim}
1678b52a31eSHanoh Haim
1688b52a31eSHanoh Haim/**
1698b52a31eSHanoh Haim * This is the irqcontrol callback to be registered to uio_info.
1708b52a31eSHanoh Haim * It can be used to disable/enable interrupt from user space processes.
1718b52a31eSHanoh Haim *
1728b52a31eSHanoh Haim * @param info
1738b52a31eSHanoh Haim *  pointer to uio_info.
1748b52a31eSHanoh Haim * @param irq_state
1758b52a31eSHanoh Haim *  state value. 1 to enable interrupt, 0 to disable interrupt.
1768b52a31eSHanoh Haim *
1778b52a31eSHanoh Haim * @return
1788b52a31eSHanoh Haim *  - On success, 0.
1798b52a31eSHanoh Haim *  - On failure, a negative value.
1808b52a31eSHanoh Haim */
1818b52a31eSHanoh Haimstatic int
1828b52a31eSHanoh Haimigbuio_pci_irqcontrol(struct uio_info *info, s32 irq_state)
1838b52a31eSHanoh Haim{
1840400a950Simarom	struct rte_uio_pci_dev *udev = info->priv;
1858b52a31eSHanoh Haim	struct pci_dev *pdev = udev->pdev;
1868b52a31eSHanoh Haim
1878b52a31eSHanoh Haim	pci_cfg_access_lock(pdev);
1888b52a31eSHanoh Haim	if (udev->mode == RTE_INTR_MODE_LEGACY)
1898b52a31eSHanoh Haim		pci_intx(pdev, !!irq_state);
1908b52a31eSHanoh Haim
1918b52a31eSHanoh Haim	else if (udev->mode == RTE_INTR_MODE_MSIX) {
1928b52a31eSHanoh Haim		struct msi_desc *desc;
1938b52a31eSHanoh Haim
1940400a950Simarom#if (LINUX_VERSION_CODE < KERNEL_VERSION(4, 3, 0))
1958b52a31eSHanoh Haim		list_for_each_entry(desc, &pdev->msi_list, list)
1968b52a31eSHanoh Haim			igbuio_msix_mask_irq(desc, irq_state);
1970400a950Simarom#else
1980400a950Simarom		list_for_each_entry(desc, &pdev->dev.msi_list, list)
1990400a950Simarom			igbuio_msix_mask_irq(desc, irq_state);
2000400a950Simarom#endif
2018b52a31eSHanoh Haim	}
2028b52a31eSHanoh Haim	pci_cfg_access_unlock(pdev);
2038b52a31eSHanoh Haim
2048b52a31eSHanoh Haim	return 0;
2058b52a31eSHanoh Haim}
2068b52a31eSHanoh Haim
2078b52a31eSHanoh Haim/**
2088b52a31eSHanoh Haim * This is interrupt handler which will check if the interrupt is for the right device.
2098b52a31eSHanoh Haim * If yes, disable it here and will be enable later.
2108b52a31eSHanoh Haim */
2118b52a31eSHanoh Haimstatic irqreturn_t
2128b52a31eSHanoh Haimigbuio_pci_irqhandler(int irq, struct uio_info *info)
2138b52a31eSHanoh Haim{
2140400a950Simarom	struct rte_uio_pci_dev *udev = info->priv;
2158b52a31eSHanoh Haim
2168b52a31eSHanoh Haim	/* Legacy mode need to mask in hardware */
2178b52a31eSHanoh Haim	if (udev->mode == RTE_INTR_MODE_LEGACY &&
2188b52a31eSHanoh Haim	    !pci_check_and_mask_intx(udev->pdev))
2198b52a31eSHanoh Haim		return IRQ_NONE;
2208b52a31eSHanoh Haim
2218b52a31eSHanoh Haim	/* Message signal mode, no share IRQ and automasked */
2228b52a31eSHanoh Haim	return IRQ_HANDLED;
2238b52a31eSHanoh Haim}
2248b52a31eSHanoh Haim
2258b52a31eSHanoh Haim#ifdef CONFIG_XEN_DOM0
2268b52a31eSHanoh Haimstatic int
2278b52a31eSHanoh Haimigbuio_dom0_mmap_phys(struct uio_info *info, struct vm_area_struct *vma)
2288b52a31eSHanoh Haim{
2298b52a31eSHanoh Haim	int idx;
2308b52a31eSHanoh Haim
2318b52a31eSHanoh Haim	idx = (int)vma->vm_pgoff;
2328b52a31eSHanoh Haim	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
2338b52a31eSHanoh Haim#ifdef HAVE_PTE_MASK_PAGE_IOMAP
2348b52a31eSHanoh Haim	vma->vm_page_prot.pgprot |= _PAGE_IOMAP;
2358b52a31eSHanoh Haim#endif
2368b52a31eSHanoh Haim
2378b52a31eSHanoh Haim	return remap_pfn_range(vma,
2388b52a31eSHanoh Haim			vma->vm_start,
2398b52a31eSHanoh Haim			info->mem[idx].addr >> PAGE_SHIFT,
2408b52a31eSHanoh Haim			vma->vm_end - vma->vm_start,
2418b52a31eSHanoh Haim			vma->vm_page_prot);
2428b52a31eSHanoh Haim}
2438b52a31eSHanoh Haim
2448b52a31eSHanoh Haim/**
2458b52a31eSHanoh Haim * This is uio device mmap method which will use igbuio mmap for Xen
2468b52a31eSHanoh Haim * Dom0 environment.
2478b52a31eSHanoh Haim */
2488b52a31eSHanoh Haimstatic int
2498b52a31eSHanoh Haimigbuio_dom0_pci_mmap(struct uio_info *info, struct vm_area_struct *vma)
2508b52a31eSHanoh Haim{
2518b52a31eSHanoh Haim	int idx;
2528b52a31eSHanoh Haim
2538b52a31eSHanoh Haim	if (vma->vm_pgoff >= MAX_UIO_MAPS)
2548b52a31eSHanoh Haim		return -EINVAL;
2558b52a31eSHanoh Haim
2568b52a31eSHanoh Haim	if (info->mem[vma->vm_pgoff].size == 0)
2578b52a31eSHanoh Haim		return -EINVAL;
2588b52a31eSHanoh Haim
2598b52a31eSHanoh Haim	idx = (int)vma->vm_pgoff;
2608b52a31eSHanoh Haim	switch (info->mem[idx].memtype) {
2618b52a31eSHanoh Haim	case UIO_MEM_PHYS:
2628b52a31eSHanoh Haim		return igbuio_dom0_mmap_phys(info, vma);
2638b52a31eSHanoh Haim	case UIO_MEM_LOGICAL:
2648b52a31eSHanoh Haim	case UIO_MEM_VIRTUAL:
2658b52a31eSHanoh Haim	default:
2668b52a31eSHanoh Haim		return -EINVAL;
2678b52a31eSHanoh Haim	}
2688b52a31eSHanoh Haim}
2698b52a31eSHanoh Haim#endif
2708b52a31eSHanoh Haim
2718b52a31eSHanoh Haim/* Remap pci resources described by bar #pci_bar in uio resource n. */
2728b52a31eSHanoh Haimstatic int
2738b52a31eSHanoh Haimigbuio_pci_setup_iomem(struct pci_dev *dev, struct uio_info *info,
2748b52a31eSHanoh Haim		       int n, int pci_bar, const char *name)
2758b52a31eSHanoh Haim{
2768b52a31eSHanoh Haim	unsigned long addr, len;
2778b52a31eSHanoh Haim	void *internal_addr;
2788b52a31eSHanoh Haim
2790400a950Simarom	if (n >= ARRAY_SIZE(info->mem))
2808b52a31eSHanoh Haim		return -EINVAL;
2818b52a31eSHanoh Haim
2828b52a31eSHanoh Haim	addr = pci_resource_start(dev, pci_bar);
2838b52a31eSHanoh Haim	len = pci_resource_len(dev, pci_bar);
2848b52a31eSHanoh Haim	if (addr == 0 || len == 0)
2858b52a31eSHanoh Haim		return -1;
2868b52a31eSHanoh Haim	internal_addr = ioremap(addr, len);
2878b52a31eSHanoh Haim	if (internal_addr == NULL)
2888b52a31eSHanoh Haim		return -1;
2898b52a31eSHanoh Haim	info->mem[n].name = name;
2908b52a31eSHanoh Haim	info->mem[n].addr = addr;
2918b52a31eSHanoh Haim	info->mem[n].internal_addr = internal_addr;
2928b52a31eSHanoh Haim	info->mem[n].size = len;
2938b52a31eSHanoh Haim	info->mem[n].memtype = UIO_MEM_PHYS;
2948b52a31eSHanoh Haim	return 0;
2958b52a31eSHanoh Haim}
2968b52a31eSHanoh Haim
2978b52a31eSHanoh Haim/* Get pci port io resources described by bar #pci_bar in uio resource n. */
2988b52a31eSHanoh Haimstatic int
2998b52a31eSHanoh Haimigbuio_pci_setup_ioport(struct pci_dev *dev, struct uio_info *info,
3008b52a31eSHanoh Haim		int n, int pci_bar, const char *name)
3018b52a31eSHanoh Haim{
3028b52a31eSHanoh Haim	unsigned long addr, len;
3038b52a31eSHanoh Haim
3040400a950Simarom	if (n >= ARRAY_SIZE(info->port))
3058b52a31eSHanoh Haim		return -EINVAL;
3068b52a31eSHanoh Haim
3078b52a31eSHanoh Haim	addr = pci_resource_start(dev, pci_bar);
3088b52a31eSHanoh Haim	len = pci_resource_len(dev, pci_bar);
3098b52a31eSHanoh Haim	if (addr == 0 || len == 0)
3108b52a31eSHanoh Haim		return -EINVAL;
3118b52a31eSHanoh Haim
3128b52a31eSHanoh Haim	info->port[n].name = name;
3138b52a31eSHanoh Haim	info->port[n].start = addr;
3148b52a31eSHanoh Haim	info->port[n].size = len;
3158b52a31eSHanoh Haim	info->port[n].porttype = UIO_PORT_X86;
3168b52a31eSHanoh Haim
3178b52a31eSHanoh Haim	return 0;
3188b52a31eSHanoh Haim}
3198b52a31eSHanoh Haim
3208b52a31eSHanoh Haim/* Unmap previously ioremap'd resources */
3218b52a31eSHanoh Haimstatic void
3228b52a31eSHanoh Haimigbuio_pci_release_iomem(struct uio_info *info)
3238b52a31eSHanoh Haim{
3248b52a31eSHanoh Haim	int i;
3258b52a31eSHanoh Haim
3268b52a31eSHanoh Haim	for (i = 0; i < MAX_UIO_MAPS; i++) {
3278b52a31eSHanoh Haim		if (info->mem[i].internal_addr)
3288b52a31eSHanoh Haim			iounmap(info->mem[i].internal_addr);
3298b52a31eSHanoh Haim	}
3308b52a31eSHanoh Haim}
3318b52a31eSHanoh Haim
3328b52a31eSHanoh Haimstatic int
3338b52a31eSHanoh Haimigbuio_setup_bars(struct pci_dev *dev, struct uio_info *info)
3348b52a31eSHanoh Haim{
3358b52a31eSHanoh Haim	int i, iom, iop, ret;
3368b52a31eSHanoh Haim	unsigned long flags;
3378b52a31eSHanoh Haim	static const char *bar_names[PCI_STD_RESOURCE_END + 1]  = {
3388b52a31eSHanoh Haim		"BAR0",
3398b52a31eSHanoh Haim		"BAR1",
3408b52a31eSHanoh Haim		"BAR2",
3418b52a31eSHanoh Haim		"BAR3",
3428b52a31eSHanoh Haim		"BAR4",
3438b52a31eSHanoh Haim		"BAR5",
3448b52a31eSHanoh Haim	};
3458b52a31eSHanoh Haim
3468b52a31eSHanoh Haim	iom = 0;
3478b52a31eSHanoh Haim	iop = 0;
3488b52a31eSHanoh Haim
3490400a950Simarom	for (i = 0; i < ARRAY_SIZE(bar_names); i++) {
3508b52a31eSHanoh Haim		if (pci_resource_len(dev, i) != 0 &&
3518b52a31eSHanoh Haim				pci_resource_start(dev, i) != 0) {
3528b52a31eSHanoh Haim			flags = pci_resource_flags(dev, i);
3538b52a31eSHanoh Haim			if (flags & IORESOURCE_MEM) {
3548b52a31eSHanoh Haim				ret = igbuio_pci_setup_iomem(dev, info, iom,
3558b52a31eSHanoh Haim							     i, bar_names[i]);
3568b52a31eSHanoh Haim				if (ret != 0)
3578b52a31eSHanoh Haim					return ret;
3588b52a31eSHanoh Haim				iom++;
3598b52a31eSHanoh Haim			} else if (flags & IORESOURCE_IO) {
3608b52a31eSHanoh Haim				ret = igbuio_pci_setup_ioport(dev, info, iop,
3618b52a31eSHanoh Haim							      i, bar_names[i]);
3628b52a31eSHanoh Haim				if (ret != 0)
3638b52a31eSHanoh Haim					return ret;
3648b52a31eSHanoh Haim				iop++;
3658b52a31eSHanoh Haim			}
3668b52a31eSHanoh Haim		}
3678b52a31eSHanoh Haim	}
3688b52a31eSHanoh Haim
3698b52a31eSHanoh Haim	return (iom != 0) ? ret : -ENOENT;
3708b52a31eSHanoh Haim}
3718b52a31eSHanoh Haim
3728b52a31eSHanoh Haim#if LINUX_VERSION_CODE < KERNEL_VERSION(3, 8, 0)
3738b52a31eSHanoh Haimstatic int __devinit
3748b52a31eSHanoh Haim#else
3758b52a31eSHanoh Haimstatic int
3768b52a31eSHanoh Haim#endif
3778b52a31eSHanoh Haimigbuio_pci_probe(struct pci_dev *dev, const struct pci_device_id *id)
3788b52a31eSHanoh Haim{
3798b52a31eSHanoh Haim	struct rte_uio_pci_dev *udev;
3808b52a31eSHanoh Haim	struct msix_entry msix_entry;
3818b52a31eSHanoh Haim	int err;
3828b52a31eSHanoh Haim
3838b52a31eSHanoh Haim	udev = kzalloc(sizeof(struct rte_uio_pci_dev), GFP_KERNEL);
3848b52a31eSHanoh Haim	if (!udev)
3858b52a31eSHanoh Haim		return -ENOMEM;
3868b52a31eSHanoh Haim
3878b52a31eSHanoh Haim	/*
3888b52a31eSHanoh Haim	 * enable device: ask low-level code to enable I/O and
3898b52a31eSHanoh Haim	 * memory
3908b52a31eSHanoh Haim	 */
3918b52a31eSHanoh Haim	err = pci_enable_device(dev);
3928b52a31eSHanoh Haim	if (err != 0) {
3938b52a31eSHanoh Haim		dev_err(&dev->dev, "Cannot enable PCI device\n");
3948b52a31eSHanoh Haim		goto fail_free;
3958b52a31eSHanoh Haim	}
3968b52a31eSHanoh Haim
3978b52a31eSHanoh Haim	/*
3988b52a31eSHanoh Haim	 * reserve device's PCI memory regions for use by this
3998b52a31eSHanoh Haim	 * module
4008b52a31eSHanoh Haim	 */
4018b52a31eSHanoh Haim	err = pci_request_regions(dev, "igb_uio");
4028b52a31eSHanoh Haim	if (err != 0) {
4038b52a31eSHanoh Haim		dev_err(&dev->dev, "Cannot request regions\n");
4048b52a31eSHanoh Haim		goto fail_disable;
4058b52a31eSHanoh Haim	}
4068b52a31eSHanoh Haim
4078b52a31eSHanoh Haim	/* enable bus mastering on the device */
4088b52a31eSHanoh Haim	pci_set_master(dev);
4098b52a31eSHanoh Haim
4108b52a31eSHanoh Haim	/* remap IO memory */
4118b52a31eSHanoh Haim	err = igbuio_setup_bars(dev, &udev->info);
4128b52a31eSHanoh Haim	if (err != 0)
4138b52a31eSHanoh Haim		goto fail_release_iomem;
4148b52a31eSHanoh Haim
4158b52a31eSHanoh Haim	/* set 64-bit DMA mask */
4168b52a31eSHanoh Haim	err = pci_set_dma_mask(dev,  DMA_BIT_MASK(64));
4178b52a31eSHanoh Haim	if (err != 0) {
4188b52a31eSHanoh Haim		dev_err(&dev->dev, "Cannot set DMA mask\n");
4198b52a31eSHanoh Haim		goto fail_release_iomem;
4208b52a31eSHanoh Haim	}
4218b52a31eSHanoh Haim
4228b52a31eSHanoh Haim	err = pci_set_consistent_dma_mask(dev, DMA_BIT_MASK(64));
4238b52a31eSHanoh Haim	if (err != 0) {
4248b52a31eSHanoh Haim		dev_err(&dev->dev, "Cannot set consistent DMA mask\n");
4258b52a31eSHanoh Haim		goto fail_release_iomem;
4268b52a31eSHanoh Haim	}
4278b52a31eSHanoh Haim
4288b52a31eSHanoh Haim	/* fill uio infos */
4298b52a31eSHanoh Haim	udev->info.name = "igb_uio";
4308b52a31eSHanoh Haim	udev->info.version = "0.1";
4318b52a31eSHanoh Haim	udev->info.handler = igbuio_pci_irqhandler;
4328b52a31eSHanoh Haim	udev->info.irqcontrol = igbuio_pci_irqcontrol;
4338b52a31eSHanoh Haim#ifdef CONFIG_XEN_DOM0
4348b52a31eSHanoh Haim	/* check if the driver run on Xen Dom0 */
4358b52a31eSHanoh Haim	if (xen_initial_domain())
4368b52a31eSHanoh Haim		udev->info.mmap = igbuio_dom0_pci_mmap;
4378b52a31eSHanoh Haim#endif
4388b52a31eSHanoh Haim	udev->info.priv = udev;
4398b52a31eSHanoh Haim	udev->pdev = dev;
4408b52a31eSHanoh Haim
4418b52a31eSHanoh Haim	switch (igbuio_intr_mode_preferred) {
4428b52a31eSHanoh Haim	case RTE_INTR_MODE_MSIX:
4438b52a31eSHanoh Haim		/* Only 1 msi-x vector needed */
4448b52a31eSHanoh Haim		msix_entry.entry = 0;
4458b52a31eSHanoh Haim		if (pci_enable_msix(dev, &msix_entry, 1) == 0) {
4468b52a31eSHanoh Haim			dev_dbg(&dev->dev, "using MSI-X");
4478b52a31eSHanoh Haim			udev->info.irq = msix_entry.vector;
4488b52a31eSHanoh Haim			udev->mode = RTE_INTR_MODE_MSIX;
4498b52a31eSHanoh Haim			break;
4508b52a31eSHanoh Haim		}
4518b52a31eSHanoh Haim		/* fall back to INTX */
4528b52a31eSHanoh Haim	case RTE_INTR_MODE_LEGACY:
4538b52a31eSHanoh Haim		if (pci_intx_mask_supported(dev)) {
4548b52a31eSHanoh Haim			dev_dbg(&dev->dev, "using INTX");
4558b52a31eSHanoh Haim			udev->info.irq_flags = IRQF_SHARED;
4568b52a31eSHanoh Haim			udev->info.irq = dev->irq;
4578b52a31eSHanoh Haim			udev->mode = RTE_INTR_MODE_LEGACY;
4588b52a31eSHanoh Haim			break;
4598b52a31eSHanoh Haim		}
4608b52a31eSHanoh Haim		dev_notice(&dev->dev, "PCI INTX mask not supported\n");
4618b52a31eSHanoh Haim		/* fall back to no IRQ */
4628b52a31eSHanoh Haim	case RTE_INTR_MODE_NONE:
4638b52a31eSHanoh Haim		udev->mode = RTE_INTR_MODE_NONE;
4648b52a31eSHanoh Haim		udev->info.irq = 0;
4658b52a31eSHanoh Haim		break;
4668b52a31eSHanoh Haim
4678b52a31eSHanoh Haim	default:
4688b52a31eSHanoh Haim		dev_err(&dev->dev, "invalid IRQ mode %u",
4698b52a31eSHanoh Haim			igbuio_intr_mode_preferred);
4708b52a31eSHanoh Haim		err = -EINVAL;
4718b52a31eSHanoh Haim		goto fail_release_iomem;
4728b52a31eSHanoh Haim	}
4738b52a31eSHanoh Haim
4748b52a31eSHanoh Haim	err = sysfs_create_group(&dev->dev.kobj, &dev_attr_grp);
4758b52a31eSHanoh Haim	if (err != 0)
4768b52a31eSHanoh Haim		goto fail_release_iomem;
4778b52a31eSHanoh Haim
4788b52a31eSHanoh Haim	/* register uio driver */
4798b52a31eSHanoh Haim	err = uio_register_device(&dev->dev, &udev->info);
4808b52a31eSHanoh Haim	if (err != 0)
4818b52a31eSHanoh Haim		goto fail_remove_group;
4828b52a31eSHanoh Haim
4838b52a31eSHanoh Haim	pci_set_drvdata(dev, udev);
4848b52a31eSHanoh Haim
4858b52a31eSHanoh Haim	dev_info(&dev->dev, "uio device registered with irq %lx\n",
4868b52a31eSHanoh Haim		 udev->info.irq);
4878b52a31eSHanoh Haim
4888b52a31eSHanoh Haim	return 0;
4898b52a31eSHanoh Haim
4908b52a31eSHanoh Haimfail_remove_group:
4918b52a31eSHanoh Haim	sysfs_remove_group(&dev->dev.kobj, &dev_attr_grp);
4928b52a31eSHanoh Haimfail_release_iomem:
4938b52a31eSHanoh Haim	igbuio_pci_release_iomem(&udev->info);
4948b52a31eSHanoh Haim	if (udev->mode == RTE_INTR_MODE_MSIX)
4958b52a31eSHanoh Haim		pci_disable_msix(udev->pdev);
4968b52a31eSHanoh Haim	pci_release_regions(dev);
4978b52a31eSHanoh Haimfail_disable:
4988b52a31eSHanoh Haim	pci_disable_device(dev);
4998b52a31eSHanoh Haimfail_free:
5008b52a31eSHanoh Haim	kfree(udev);
5018b52a31eSHanoh Haim
5028b52a31eSHanoh Haim	return err;
5038b52a31eSHanoh Haim}
5048b52a31eSHanoh Haim
5058b52a31eSHanoh Haimstatic void
5068b52a31eSHanoh Haimigbuio_pci_remove(struct pci_dev *dev)
5078b52a31eSHanoh Haim{
5080400a950Simarom	struct rte_uio_pci_dev *udev = pci_get_drvdata(dev);
5098b52a31eSHanoh Haim
5108b52a31eSHanoh Haim	sysfs_remove_group(&dev->dev.kobj, &dev_attr_grp);
5110400a950Simarom	uio_unregister_device(&udev->info);
5120400a950Simarom	igbuio_pci_release_iomem(&udev->info);
5138b52a31eSHanoh Haim	if (udev->mode == RTE_INTR_MODE_MSIX)
5148b52a31eSHanoh Haim		pci_disable_msix(dev);
5158b52a31eSHanoh Haim	pci_release_regions(dev);
5168b52a31eSHanoh Haim	pci_disable_device(dev);
5178b52a31eSHanoh Haim	pci_set_drvdata(dev, NULL);
5180400a950Simarom	kfree(udev);
5198b52a31eSHanoh Haim}
5208b52a31eSHanoh Haim
5218b52a31eSHanoh Haimstatic int
5228b52a31eSHanoh Haimigbuio_config_intr_mode(char *intr_str)
5238b52a31eSHanoh Haim{
5248b52a31eSHanoh Haim	if (!intr_str) {
5258b52a31eSHanoh Haim		pr_info("Use MSIX interrupt by default\n");
5268b52a31eSHanoh Haim		return 0;
5278b52a31eSHanoh Haim	}
5288b52a31eSHanoh Haim
5298b52a31eSHanoh Haim	if (!strcmp(intr_str, RTE_INTR_MODE_MSIX_NAME)) {
5308b52a31eSHanoh Haim		igbuio_intr_mode_preferred = RTE_INTR_MODE_MSIX;
5318b52a31eSHanoh Haim		pr_info("Use MSIX interrupt\n");
5328b52a31eSHanoh Haim	} else if (!strcmp(intr_str, RTE_INTR_MODE_LEGACY_NAME)) {
5338b52a31eSHanoh Haim		igbuio_intr_mode_preferred = RTE_INTR_MODE_LEGACY;
5348b52a31eSHanoh Haim		pr_info("Use legacy interrupt\n");
5358b52a31eSHanoh Haim	} else {
5368b52a31eSHanoh Haim		pr_info("Error: bad parameter - %s\n", intr_str);
5378b52a31eSHanoh Haim		return -EINVAL;
5388b52a31eSHanoh Haim	}
5398b52a31eSHanoh Haim
5408b52a31eSHanoh Haim	return 0;
5418b52a31eSHanoh Haim}
5428b52a31eSHanoh Haim
5438b52a31eSHanoh Haimstatic struct pci_driver igbuio_pci_driver = {
5448b52a31eSHanoh Haim	.name = "igb_uio",
5458b52a31eSHanoh Haim	.id_table = NULL,
5468b52a31eSHanoh Haim	.probe = igbuio_pci_probe,
5478b52a31eSHanoh Haim	.remove = igbuio_pci_remove,
5488b52a31eSHanoh Haim};
5498b52a31eSHanoh Haim
5508b52a31eSHanoh Haimstatic int __init
5518b52a31eSHanoh Haimigbuio_pci_init_module(void)
5528b52a31eSHanoh Haim{
5538b52a31eSHanoh Haim	int ret;
5548b52a31eSHanoh Haim
5558b52a31eSHanoh Haim	ret = igbuio_config_intr_mode(intr_mode);
5568b52a31eSHanoh Haim	if (ret < 0)
5578b52a31eSHanoh Haim		return ret;
5588b52a31eSHanoh Haim
5598b52a31eSHanoh Haim	return pci_register_driver(&igbuio_pci_driver);
5608b52a31eSHanoh Haim}
5618b52a31eSHanoh Haim
5628b52a31eSHanoh Haimstatic void __exit
5638b52a31eSHanoh Haimigbuio_pci_exit_module(void)
5648b52a31eSHanoh Haim{
5658b52a31eSHanoh Haim	pci_unregister_driver(&igbuio_pci_driver);
5668b52a31eSHanoh Haim}
5678b52a31eSHanoh Haim
5688b52a31eSHanoh Haimmodule_init(igbuio_pci_init_module);
5698b52a31eSHanoh Haimmodule_exit(igbuio_pci_exit_module);
5708b52a31eSHanoh Haim
5718b52a31eSHanoh Haimmodule_param(intr_mode, charp, S_IRUGO);
5728b52a31eSHanoh HaimMODULE_PARM_DESC(intr_mode,
5738b52a31eSHanoh Haim"igb_uio interrupt mode (default=msix):\n"
5748b52a31eSHanoh Haim"    " RTE_INTR_MODE_MSIX_NAME "       Use MSIX interrupt\n"
5758b52a31eSHanoh Haim"    " RTE_INTR_MODE_LEGACY_NAME "     Use Legacy interrupt\n"
5768b52a31eSHanoh Haim"\n");
5778b52a31eSHanoh Haim
5788b52a31eSHanoh HaimMODULE_DESCRIPTION("UIO driver for Intel IGB PCI cards");
5798b52a31eSHanoh HaimMODULE_LICENSE("GPL");
5808b52a31eSHanoh HaimMODULE_AUTHOR("Intel Corporation");
581