1123 lines · c
1// SPDX-License-Identifier: GPL-2.02/*3 * Volume Management Device driver4 * Copyright (c) 2015, Intel Corporation.5 */6 7#include <linux/device.h>8#include <linux/interrupt.h>9#include <linux/irq.h>10#include <linux/kernel.h>11#include <linux/module.h>12#include <linux/msi.h>13#include <linux/pci.h>14#include <linux/pci-acpi.h>15#include <linux/pci-ecam.h>16#include <linux/srcu.h>17#include <linux/rculist.h>18#include <linux/rcupdate.h>19 20#include <asm/irqdomain.h>21 22#define VMD_CFGBAR 023#define VMD_MEMBAR1 224#define VMD_MEMBAR2 425 26#define PCI_REG_VMCAP 0x4027#define BUS_RESTRICT_CAP(vmcap) (vmcap & 0x1)28#define PCI_REG_VMCONFIG 0x4429#define BUS_RESTRICT_CFG(vmcfg) ((vmcfg >> 8) & 0x3)30#define VMCONFIG_MSI_REMAP 0x231#define PCI_REG_VMLOCK 0x7032#define MB2_SHADOW_EN(vmlock) (vmlock & 0x2)33 34#define MB2_SHADOW_OFFSET 0x200035#define MB2_SHADOW_SIZE 1636 37enum vmd_features {38 /*39 * Device may contain registers which hint the physical location of the40 * membars, in order to allow proper address translation during41 * resource assignment to enable guest virtualization42 */43 VMD_FEAT_HAS_MEMBAR_SHADOW = (1 << 0),44 45 /*46 * Device may provide root port configuration information which limits47 * bus numbering48 */49 VMD_FEAT_HAS_BUS_RESTRICTIONS = (1 << 1),50 51 /*52 * Device contains physical location shadow registers in53 * vendor-specific capability space54 */55 VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP = (1 << 2),56 57 /*58 * Device may use MSI-X vector 0 for software triggering and will not59 * be used for MSI remapping60 */61 VMD_FEAT_OFFSET_FIRST_VECTOR = (1 << 3),62 63 /*64 * Device can bypass remapping MSI-X transactions into its MSI-X table,65 * avoiding the requirement of a VMD MSI domain for child device66 * interrupt handling.67 */68 VMD_FEAT_CAN_BYPASS_MSI_REMAP = (1 << 4),69 70 /*71 * Enable ASPM on the PCIE root ports and set the default LTR of the72 * storage devices on platforms where these values are not configured by73 * BIOS. This is needed for laptops, which require these settings for74 * proper power management of the SoC.75 */76 VMD_FEAT_BIOS_PM_QUIRK = (1 << 5),77};78 79#define VMD_BIOS_PM_QUIRK_LTR 0x1003 /* 3145728 ns */80 81#define VMD_FEATS_CLIENT (VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP | \82 VMD_FEAT_HAS_BUS_RESTRICTIONS | \83 VMD_FEAT_OFFSET_FIRST_VECTOR | \84 VMD_FEAT_BIOS_PM_QUIRK)85 86static DEFINE_IDA(vmd_instance_ida);87 88/*89 * Lock for manipulating VMD IRQ lists.90 */91static DEFINE_RAW_SPINLOCK(list_lock);92 93/**94 * struct vmd_irq - private data to map driver IRQ to the VMD shared vector95 * @node: list item for parent traversal.96 * @irq: back pointer to parent.97 * @enabled: true if driver enabled IRQ98 * @virq: the virtual IRQ value provided to the requesting driver.99 *100 * Every MSI/MSI-X IRQ requested for a device in a VMD domain will be mapped to101 * a VMD IRQ using this structure.102 */103struct vmd_irq {104 struct list_head node;105 struct vmd_irq_list *irq;106 bool enabled;107 unsigned int virq;108};109 110/**111 * struct vmd_irq_list - list of driver requested IRQs mapping to a VMD vector112 * @irq_list: the list of irq's the VMD one demuxes to.113 * @srcu: SRCU struct for local synchronization.114 * @count: number of child IRQs assigned to this vector; used to track115 * sharing.116 * @virq: The underlying VMD Linux interrupt number117 */118struct vmd_irq_list {119 struct list_head irq_list;120 struct srcu_struct srcu;121 unsigned int count;122 unsigned int virq;123};124 125struct vmd_dev {126 struct pci_dev *dev;127 128 spinlock_t cfg_lock;129 void __iomem *cfgbar;130 131 int msix_count;132 struct vmd_irq_list *irqs;133 134 struct pci_sysdata sysdata;135 struct resource resources[3];136 struct irq_domain *irq_domain;137 struct pci_bus *bus;138 u8 busn_start;139 u8 first_vec;140 char *name;141 int instance;142};143 144static inline struct vmd_dev *vmd_from_bus(struct pci_bus *bus)145{146 return container_of(bus->sysdata, struct vmd_dev, sysdata);147}148 149static inline unsigned int index_from_irqs(struct vmd_dev *vmd,150 struct vmd_irq_list *irqs)151{152 return irqs - vmd->irqs;153}154 155/*156 * Drivers managing a device in a VMD domain allocate their own IRQs as before,157 * but the MSI entry for the hardware it's driving will be programmed with a158 * destination ID for the VMD MSI-X table. The VMD muxes interrupts in its159 * domain into one of its own, and the VMD driver de-muxes these for the160 * handlers sharing that VMD IRQ. The vmd irq_domain provides the operations161 * and irq_chip to set this up.162 */163static void vmd_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)164{165 struct vmd_irq *vmdirq = data->chip_data;166 struct vmd_irq_list *irq = vmdirq->irq;167 struct vmd_dev *vmd = irq_data_get_irq_handler_data(data);168 169 memset(msg, 0, sizeof(*msg));170 msg->address_hi = X86_MSI_BASE_ADDRESS_HIGH;171 msg->arch_addr_lo.base_address = X86_MSI_BASE_ADDRESS_LOW;172 msg->arch_addr_lo.destid_0_7 = index_from_irqs(vmd, irq);173}174 175/*176 * We rely on MSI_FLAG_USE_DEF_CHIP_OPS to set the IRQ mask/unmask ops.177 */178static void vmd_irq_enable(struct irq_data *data)179{180 struct vmd_irq *vmdirq = data->chip_data;181 unsigned long flags;182 183 raw_spin_lock_irqsave(&list_lock, flags);184 WARN_ON(vmdirq->enabled);185 list_add_tail_rcu(&vmdirq->node, &vmdirq->irq->irq_list);186 vmdirq->enabled = true;187 raw_spin_unlock_irqrestore(&list_lock, flags);188 189 data->chip->irq_unmask(data);190}191 192static void vmd_irq_disable(struct irq_data *data)193{194 struct vmd_irq *vmdirq = data->chip_data;195 unsigned long flags;196 197 data->chip->irq_mask(data);198 199 raw_spin_lock_irqsave(&list_lock, flags);200 if (vmdirq->enabled) {201 list_del_rcu(&vmdirq->node);202 vmdirq->enabled = false;203 }204 raw_spin_unlock_irqrestore(&list_lock, flags);205}206 207static struct irq_chip vmd_msi_controller = {208 .name = "VMD-MSI",209 .irq_enable = vmd_irq_enable,210 .irq_disable = vmd_irq_disable,211 .irq_compose_msi_msg = vmd_compose_msi_msg,212};213 214static irq_hw_number_t vmd_get_hwirq(struct msi_domain_info *info,215 msi_alloc_info_t *arg)216{217 return 0;218}219 220/*221 * XXX: We can be even smarter selecting the best IRQ once we solve the222 * affinity problem.223 */224static struct vmd_irq_list *vmd_next_irq(struct vmd_dev *vmd, struct msi_desc *desc)225{226 unsigned long flags;227 int i, best;228 229 if (vmd->msix_count == 1 + vmd->first_vec)230 return &vmd->irqs[vmd->first_vec];231 232 /*233 * White list for fast-interrupt handlers. All others will share the234 * "slow" interrupt vector.235 */236 switch (msi_desc_to_pci_dev(desc)->class) {237 case PCI_CLASS_STORAGE_EXPRESS:238 break;239 default:240 return &vmd->irqs[vmd->first_vec];241 }242 243 raw_spin_lock_irqsave(&list_lock, flags);244 best = vmd->first_vec + 1;245 for (i = best; i < vmd->msix_count; i++)246 if (vmd->irqs[i].count < vmd->irqs[best].count)247 best = i;248 vmd->irqs[best].count++;249 raw_spin_unlock_irqrestore(&list_lock, flags);250 251 return &vmd->irqs[best];252}253 254static int vmd_msi_init(struct irq_domain *domain, struct msi_domain_info *info,255 unsigned int virq, irq_hw_number_t hwirq,256 msi_alloc_info_t *arg)257{258 struct msi_desc *desc = arg->desc;259 struct vmd_dev *vmd = vmd_from_bus(msi_desc_to_pci_dev(desc)->bus);260 struct vmd_irq *vmdirq = kzalloc(sizeof(*vmdirq), GFP_KERNEL);261 262 if (!vmdirq)263 return -ENOMEM;264 265 INIT_LIST_HEAD(&vmdirq->node);266 vmdirq->irq = vmd_next_irq(vmd, desc);267 vmdirq->virq = virq;268 269 irq_domain_set_info(domain, virq, vmdirq->irq->virq, info->chip, vmdirq,270 handle_untracked_irq, vmd, NULL);271 return 0;272}273 274static void vmd_msi_free(struct irq_domain *domain,275 struct msi_domain_info *info, unsigned int virq)276{277 struct vmd_irq *vmdirq = irq_get_chip_data(virq);278 unsigned long flags;279 280 synchronize_srcu(&vmdirq->irq->srcu);281 282 /* XXX: Potential optimization to rebalance */283 raw_spin_lock_irqsave(&list_lock, flags);284 vmdirq->irq->count--;285 raw_spin_unlock_irqrestore(&list_lock, flags);286 287 kfree(vmdirq);288}289 290static int vmd_msi_prepare(struct irq_domain *domain, struct device *dev,291 int nvec, msi_alloc_info_t *arg)292{293 struct pci_dev *pdev = to_pci_dev(dev);294 struct vmd_dev *vmd = vmd_from_bus(pdev->bus);295 296 if (nvec > vmd->msix_count)297 return vmd->msix_count;298 299 memset(arg, 0, sizeof(*arg));300 return 0;301}302 303static void vmd_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc)304{305 arg->desc = desc;306}307 308static struct msi_domain_ops vmd_msi_domain_ops = {309 .get_hwirq = vmd_get_hwirq,310 .msi_init = vmd_msi_init,311 .msi_free = vmd_msi_free,312 .msi_prepare = vmd_msi_prepare,313 .set_desc = vmd_set_desc,314};315 316static struct msi_domain_info vmd_msi_domain_info = {317 .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |318 MSI_FLAG_NO_AFFINITY | MSI_FLAG_PCI_MSIX,319 .ops = &vmd_msi_domain_ops,320 .chip = &vmd_msi_controller,321};322 323static void vmd_set_msi_remapping(struct vmd_dev *vmd, bool enable)324{325 u16 reg;326 327 pci_read_config_word(vmd->dev, PCI_REG_VMCONFIG, ®);328 reg = enable ? (reg & ~VMCONFIG_MSI_REMAP) :329 (reg | VMCONFIG_MSI_REMAP);330 pci_write_config_word(vmd->dev, PCI_REG_VMCONFIG, reg);331}332 333static int vmd_create_irq_domain(struct vmd_dev *vmd)334{335 struct fwnode_handle *fn;336 337 fn = irq_domain_alloc_named_id_fwnode("VMD-MSI", vmd->sysdata.domain);338 if (!fn)339 return -ENODEV;340 341 vmd->irq_domain = pci_msi_create_irq_domain(fn, &vmd_msi_domain_info, NULL);342 if (!vmd->irq_domain) {343 irq_domain_free_fwnode(fn);344 return -ENODEV;345 }346 347 return 0;348}349 350static void vmd_remove_irq_domain(struct vmd_dev *vmd)351{352 /*353 * Some production BIOS won't enable remapping between soft reboots.354 * Ensure remapping is restored before unloading the driver.355 */356 if (!vmd->msix_count)357 vmd_set_msi_remapping(vmd, true);358 359 if (vmd->irq_domain) {360 struct fwnode_handle *fn = vmd->irq_domain->fwnode;361 362 irq_domain_remove(vmd->irq_domain);363 irq_domain_free_fwnode(fn);364 }365}366 367static void __iomem *vmd_cfg_addr(struct vmd_dev *vmd, struct pci_bus *bus,368 unsigned int devfn, int reg, int len)369{370 unsigned int busnr_ecam = bus->number - vmd->busn_start;371 u32 offset = PCIE_ECAM_OFFSET(busnr_ecam, devfn, reg);372 373 if (offset + len >= resource_size(&vmd->dev->resource[VMD_CFGBAR]))374 return NULL;375 376 return vmd->cfgbar + offset;377}378 379/*380 * CPU may deadlock if config space is not serialized on some versions of this381 * hardware, so all config space access is done under a spinlock.382 */383static int vmd_pci_read(struct pci_bus *bus, unsigned int devfn, int reg,384 int len, u32 *value)385{386 struct vmd_dev *vmd = vmd_from_bus(bus);387 void __iomem *addr = vmd_cfg_addr(vmd, bus, devfn, reg, len);388 unsigned long flags;389 int ret = 0;390 391 if (!addr)392 return -EFAULT;393 394 spin_lock_irqsave(&vmd->cfg_lock, flags);395 switch (len) {396 case 1:397 *value = readb(addr);398 break;399 case 2:400 *value = readw(addr);401 break;402 case 4:403 *value = readl(addr);404 break;405 default:406 ret = -EINVAL;407 break;408 }409 spin_unlock_irqrestore(&vmd->cfg_lock, flags);410 return ret;411}412 413/*414 * VMD h/w converts non-posted config writes to posted memory writes. The415 * read-back in this function forces the completion so it returns only after416 * the config space was written, as expected.417 */418static int vmd_pci_write(struct pci_bus *bus, unsigned int devfn, int reg,419 int len, u32 value)420{421 struct vmd_dev *vmd = vmd_from_bus(bus);422 void __iomem *addr = vmd_cfg_addr(vmd, bus, devfn, reg, len);423 unsigned long flags;424 int ret = 0;425 426 if (!addr)427 return -EFAULT;428 429 spin_lock_irqsave(&vmd->cfg_lock, flags);430 switch (len) {431 case 1:432 writeb(value, addr);433 readb(addr);434 break;435 case 2:436 writew(value, addr);437 readw(addr);438 break;439 case 4:440 writel(value, addr);441 readl(addr);442 break;443 default:444 ret = -EINVAL;445 break;446 }447 spin_unlock_irqrestore(&vmd->cfg_lock, flags);448 return ret;449}450 451static struct pci_ops vmd_ops = {452 .read = vmd_pci_read,453 .write = vmd_pci_write,454};455 456#ifdef CONFIG_ACPI457static struct acpi_device *vmd_acpi_find_companion(struct pci_dev *pci_dev)458{459 struct pci_host_bridge *bridge;460 u32 busnr, addr;461 462 if (pci_dev->bus->ops != &vmd_ops)463 return NULL;464 465 bridge = pci_find_host_bridge(pci_dev->bus);466 busnr = pci_dev->bus->number - bridge->bus->number;467 /*468 * The address computation below is only applicable to relative bus469 * numbers below 32.470 */471 if (busnr > 31)472 return NULL;473 474 addr = (busnr << 24) | ((u32)pci_dev->devfn << 16) | 0x8000FFFFU;475 476 dev_dbg(&pci_dev->dev, "Looking for ACPI companion (address 0x%x)\n",477 addr);478 479 return acpi_find_child_device(ACPI_COMPANION(bridge->dev.parent), addr,480 false);481}482 483static bool hook_installed;484 485static void vmd_acpi_begin(void)486{487 if (pci_acpi_set_companion_lookup_hook(vmd_acpi_find_companion))488 return;489 490 hook_installed = true;491}492 493static void vmd_acpi_end(void)494{495 if (!hook_installed)496 return;497 498 pci_acpi_clear_companion_lookup_hook();499 hook_installed = false;500}501#else502static inline void vmd_acpi_begin(void) { }503static inline void vmd_acpi_end(void) { }504#endif /* CONFIG_ACPI */505 506static void vmd_domain_reset(struct vmd_dev *vmd)507{508 u16 bus, max_buses = resource_size(&vmd->resources[0]);509 u8 dev, functions, fn, hdr_type;510 char __iomem *base;511 512 for (bus = 0; bus < max_buses; bus++) {513 for (dev = 0; dev < 32; dev++) {514 base = vmd->cfgbar + PCIE_ECAM_OFFSET(bus,515 PCI_DEVFN(dev, 0), 0);516 517 hdr_type = readb(base + PCI_HEADER_TYPE);518 519 functions = (hdr_type & PCI_HEADER_TYPE_MFD) ? 8 : 1;520 for (fn = 0; fn < functions; fn++) {521 base = vmd->cfgbar + PCIE_ECAM_OFFSET(bus,522 PCI_DEVFN(dev, fn), 0);523 524 hdr_type = readb(base + PCI_HEADER_TYPE) &525 PCI_HEADER_TYPE_MASK;526 527 if (hdr_type != PCI_HEADER_TYPE_BRIDGE ||528 (readw(base + PCI_CLASS_DEVICE) !=529 PCI_CLASS_BRIDGE_PCI))530 continue;531 532 /*533 * Temporarily disable the I/O range before updating534 * PCI_IO_BASE.535 */536 writel(0x0000ffff, base + PCI_IO_BASE_UPPER16);537 /* Update lower 16 bits of I/O base/limit */538 writew(0x00f0, base + PCI_IO_BASE);539 /* Update upper 16 bits of I/O base/limit */540 writel(0, base + PCI_IO_BASE_UPPER16);541 542 /* MMIO Base/Limit */543 writel(0x0000fff0, base + PCI_MEMORY_BASE);544 545 /* Prefetchable MMIO Base/Limit */546 writel(0, base + PCI_PREF_LIMIT_UPPER32);547 writel(0x0000fff0, base + PCI_PREF_MEMORY_BASE);548 writel(0xffffffff, base + PCI_PREF_BASE_UPPER32);549 }550 }551 }552}553 554static void vmd_attach_resources(struct vmd_dev *vmd)555{556 vmd->dev->resource[VMD_MEMBAR1].child = &vmd->resources[1];557 vmd->dev->resource[VMD_MEMBAR2].child = &vmd->resources[2];558}559 560static void vmd_detach_resources(struct vmd_dev *vmd)561{562 vmd->dev->resource[VMD_MEMBAR1].child = NULL;563 vmd->dev->resource[VMD_MEMBAR2].child = NULL;564}565 566/*567 * VMD domains start at 0x10000 to not clash with ACPI _SEG domains.568 * Per ACPI r6.0, sec 6.5.6, _SEG returns an integer, of which the lower569 * 16 bits are the PCI Segment Group (domain) number. Other bits are570 * currently reserved.571 */572static int vmd_find_free_domain(void)573{574 int domain = 0xffff;575 struct pci_bus *bus = NULL;576 577 while ((bus = pci_find_next_bus(bus)) != NULL)578 domain = max_t(int, domain, pci_domain_nr(bus));579 return domain + 1;580}581 582static int vmd_get_phys_offsets(struct vmd_dev *vmd, bool native_hint,583 resource_size_t *offset1,584 resource_size_t *offset2)585{586 struct pci_dev *dev = vmd->dev;587 u64 phys1, phys2;588 589 if (native_hint) {590 u32 vmlock;591 int ret;592 593 ret = pci_read_config_dword(dev, PCI_REG_VMLOCK, &vmlock);594 if (ret || PCI_POSSIBLE_ERROR(vmlock))595 return -ENODEV;596 597 if (MB2_SHADOW_EN(vmlock)) {598 void __iomem *membar2;599 600 membar2 = pci_iomap(dev, VMD_MEMBAR2, 0);601 if (!membar2)602 return -ENOMEM;603 phys1 = readq(membar2 + MB2_SHADOW_OFFSET);604 phys2 = readq(membar2 + MB2_SHADOW_OFFSET + 8);605 pci_iounmap(dev, membar2);606 } else607 return 0;608 } else {609 /* Hypervisor-Emulated Vendor-Specific Capability */610 int pos = pci_find_capability(dev, PCI_CAP_ID_VNDR);611 u32 reg, regu;612 613 pci_read_config_dword(dev, pos + 4, ®);614 615 /* "SHDW" */616 if (pos && reg == 0x53484457) {617 pci_read_config_dword(dev, pos + 8, ®);618 pci_read_config_dword(dev, pos + 12, ®u);619 phys1 = (u64) regu << 32 | reg;620 621 pci_read_config_dword(dev, pos + 16, ®);622 pci_read_config_dword(dev, pos + 20, ®u);623 phys2 = (u64) regu << 32 | reg;624 } else625 return 0;626 }627 628 *offset1 = dev->resource[VMD_MEMBAR1].start -629 (phys1 & PCI_BASE_ADDRESS_MEM_MASK);630 *offset2 = dev->resource[VMD_MEMBAR2].start -631 (phys2 & PCI_BASE_ADDRESS_MEM_MASK);632 633 return 0;634}635 636static int vmd_get_bus_number_start(struct vmd_dev *vmd)637{638 struct pci_dev *dev = vmd->dev;639 u16 reg;640 641 pci_read_config_word(dev, PCI_REG_VMCAP, ®);642 if (BUS_RESTRICT_CAP(reg)) {643 pci_read_config_word(dev, PCI_REG_VMCONFIG, ®);644 645 switch (BUS_RESTRICT_CFG(reg)) {646 case 0:647 vmd->busn_start = 0;648 break;649 case 1:650 vmd->busn_start = 128;651 break;652 case 2:653 vmd->busn_start = 224;654 break;655 default:656 pci_err(dev, "Unknown Bus Offset Setting (%d)\n",657 BUS_RESTRICT_CFG(reg));658 return -ENODEV;659 }660 }661 662 return 0;663}664 665static irqreturn_t vmd_irq(int irq, void *data)666{667 struct vmd_irq_list *irqs = data;668 struct vmd_irq *vmdirq;669 int idx;670 671 idx = srcu_read_lock(&irqs->srcu);672 list_for_each_entry_rcu(vmdirq, &irqs->irq_list, node)673 generic_handle_irq(vmdirq->virq);674 srcu_read_unlock(&irqs->srcu, idx);675 676 return IRQ_HANDLED;677}678 679static int vmd_alloc_irqs(struct vmd_dev *vmd)680{681 struct pci_dev *dev = vmd->dev;682 int i, err;683 684 vmd->msix_count = pci_msix_vec_count(dev);685 if (vmd->msix_count < 0)686 return -ENODEV;687 688 vmd->msix_count = pci_alloc_irq_vectors(dev, vmd->first_vec + 1,689 vmd->msix_count, PCI_IRQ_MSIX);690 if (vmd->msix_count < 0)691 return vmd->msix_count;692 693 vmd->irqs = devm_kcalloc(&dev->dev, vmd->msix_count, sizeof(*vmd->irqs),694 GFP_KERNEL);695 if (!vmd->irqs)696 return -ENOMEM;697 698 for (i = 0; i < vmd->msix_count; i++) {699 err = init_srcu_struct(&vmd->irqs[i].srcu);700 if (err)701 return err;702 703 INIT_LIST_HEAD(&vmd->irqs[i].irq_list);704 vmd->irqs[i].virq = pci_irq_vector(dev, i);705 err = devm_request_irq(&dev->dev, vmd->irqs[i].virq,706 vmd_irq, IRQF_NO_THREAD,707 vmd->name, &vmd->irqs[i]);708 if (err)709 return err;710 }711 712 return 0;713}714 715/*716 * Since VMD is an aperture to regular PCIe root ports, only allow it to717 * control features that the OS is allowed to control on the physical PCI bus.718 */719static void vmd_copy_host_bridge_flags(struct pci_host_bridge *root_bridge,720 struct pci_host_bridge *vmd_bridge)721{722 vmd_bridge->native_pcie_hotplug = root_bridge->native_pcie_hotplug;723 vmd_bridge->native_shpc_hotplug = root_bridge->native_shpc_hotplug;724 vmd_bridge->native_aer = root_bridge->native_aer;725 vmd_bridge->native_pme = root_bridge->native_pme;726 vmd_bridge->native_ltr = root_bridge->native_ltr;727 vmd_bridge->native_dpc = root_bridge->native_dpc;728}729 730/*731 * Enable ASPM and LTR settings on devices that aren't configured by BIOS.732 */733static int vmd_pm_enable_quirk(struct pci_dev *pdev, void *userdata)734{735 unsigned long features = *(unsigned long *)userdata;736 u16 ltr = VMD_BIOS_PM_QUIRK_LTR;737 u32 ltr_reg;738 int pos;739 740 if (!(features & VMD_FEAT_BIOS_PM_QUIRK))741 return 0;742 743 pci_enable_link_state_locked(pdev, PCIE_LINK_STATE_ALL);744 745 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_LTR);746 if (!pos)747 return 0;748 749 /*750 * Skip if the max snoop LTR is non-zero, indicating BIOS has set it751 * so the LTR quirk is not needed.752 */753 pci_read_config_dword(pdev, pos + PCI_LTR_MAX_SNOOP_LAT, <r_reg);754 if (!!(ltr_reg & (PCI_LTR_VALUE_MASK | PCI_LTR_SCALE_MASK)))755 return 0;756 757 /*758 * Set the default values to the maximum required by the platform to759 * allow the deepest power management savings. Write as a DWORD where760 * the lower word is the max snoop latency and the upper word is the761 * max non-snoop latency.762 */763 ltr_reg = (ltr << 16) | ltr;764 pci_write_config_dword(pdev, pos + PCI_LTR_MAX_SNOOP_LAT, ltr_reg);765 pci_info(pdev, "VMD: Default LTR value set by driver\n");766 767 return 0;768}769 770static int vmd_enable_domain(struct vmd_dev *vmd, unsigned long features)771{772 struct pci_sysdata *sd = &vmd->sysdata;773 struct resource *res;774 u32 upper_bits;775 unsigned long flags;776 LIST_HEAD(resources);777 resource_size_t offset[2] = {0};778 resource_size_t membar2_offset = 0x2000;779 struct pci_bus *child;780 struct pci_dev *dev;781 int ret;782 783 /*784 * Shadow registers may exist in certain VMD device ids which allow785 * guests to correctly assign host physical addresses to the root ports786 * and child devices. These registers will either return the host value787 * or 0, depending on an enable bit in the VMD device.788 */789 if (features & VMD_FEAT_HAS_MEMBAR_SHADOW) {790 membar2_offset = MB2_SHADOW_OFFSET + MB2_SHADOW_SIZE;791 ret = vmd_get_phys_offsets(vmd, true, &offset[0], &offset[1]);792 if (ret)793 return ret;794 } else if (features & VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP) {795 ret = vmd_get_phys_offsets(vmd, false, &offset[0], &offset[1]);796 if (ret)797 return ret;798 }799 800 /*801 * Certain VMD devices may have a root port configuration option which802 * limits the bus range to between 0-127, 128-255, or 224-255803 */804 if (features & VMD_FEAT_HAS_BUS_RESTRICTIONS) {805 ret = vmd_get_bus_number_start(vmd);806 if (ret)807 return ret;808 }809 810 res = &vmd->dev->resource[VMD_CFGBAR];811 vmd->resources[0] = (struct resource) {812 .name = "VMD CFGBAR",813 .start = vmd->busn_start,814 .end = vmd->busn_start + (resource_size(res) >> 20) - 1,815 .flags = IORESOURCE_BUS | IORESOURCE_PCI_FIXED,816 };817 818 /*819 * If the window is below 4GB, clear IORESOURCE_MEM_64 so we can820 * put 32-bit resources in the window.821 *822 * There's no hardware reason why a 64-bit window *couldn't*823 * contain a 32-bit resource, but pbus_size_mem() computes the824 * bridge window size assuming a 64-bit window will contain no825 * 32-bit resources. __pci_assign_resource() enforces that826 * artificial restriction to make sure everything will fit.827 *828 * The only way we could use a 64-bit non-prefetchable MEMBAR is829 * if its address is <4GB so that we can convert it to a 32-bit830 * resource. To be visible to the host OS, all VMD endpoints must831 * be initially configured by platform BIOS, which includes setting832 * up these resources. We can assume the device is configured833 * according to the platform needs.834 */835 res = &vmd->dev->resource[VMD_MEMBAR1];836 upper_bits = upper_32_bits(res->end);837 flags = res->flags & ~IORESOURCE_SIZEALIGN;838 if (!upper_bits)839 flags &= ~IORESOURCE_MEM_64;840 vmd->resources[1] = (struct resource) {841 .name = "VMD MEMBAR1",842 .start = res->start,843 .end = res->end,844 .flags = flags,845 .parent = res,846 };847 848 res = &vmd->dev->resource[VMD_MEMBAR2];849 upper_bits = upper_32_bits(res->end);850 flags = res->flags & ~IORESOURCE_SIZEALIGN;851 if (!upper_bits)852 flags &= ~IORESOURCE_MEM_64;853 vmd->resources[2] = (struct resource) {854 .name = "VMD MEMBAR2",855 .start = res->start + membar2_offset,856 .end = res->end,857 .flags = flags,858 .parent = res,859 };860 861 sd->vmd_dev = vmd->dev;862 sd->domain = vmd_find_free_domain();863 if (sd->domain < 0)864 return sd->domain;865 866 sd->node = pcibus_to_node(vmd->dev->bus);867 868 /*869 * Currently MSI remapping must be enabled in guest passthrough mode870 * due to some missing interrupt remapping plumbing. This is probably871 * acceptable because the guest is usually CPU-limited and MSI872 * remapping doesn't become a performance bottleneck.873 */874 if (!(features & VMD_FEAT_CAN_BYPASS_MSI_REMAP) ||875 offset[0] || offset[1]) {876 ret = vmd_alloc_irqs(vmd);877 if (ret)878 return ret;879 880 vmd_set_msi_remapping(vmd, true);881 882 ret = vmd_create_irq_domain(vmd);883 if (ret)884 return ret;885 886 /*887 * Override the IRQ domain bus token so the domain can be888 * distinguished from a regular PCI/MSI domain.889 */890 irq_domain_update_bus_token(vmd->irq_domain, DOMAIN_BUS_VMD_MSI);891 } else {892 vmd_set_msi_remapping(vmd, false);893 }894 895 pci_add_resource(&resources, &vmd->resources[0]);896 pci_add_resource_offset(&resources, &vmd->resources[1], offset[0]);897 pci_add_resource_offset(&resources, &vmd->resources[2], offset[1]);898 899 vmd->bus = pci_create_root_bus(&vmd->dev->dev, vmd->busn_start,900 &vmd_ops, sd, &resources);901 if (!vmd->bus) {902 pci_free_resource_list(&resources);903 vmd_remove_irq_domain(vmd);904 return -ENODEV;905 }906 907 vmd_copy_host_bridge_flags(pci_find_host_bridge(vmd->dev->bus),908 to_pci_host_bridge(vmd->bus->bridge));909 910 vmd_attach_resources(vmd);911 if (vmd->irq_domain)912 dev_set_msi_domain(&vmd->bus->dev, vmd->irq_domain);913 else914 dev_set_msi_domain(&vmd->bus->dev,915 dev_get_msi_domain(&vmd->dev->dev));916 917 WARN(sysfs_create_link(&vmd->dev->dev.kobj, &vmd->bus->dev.kobj,918 "domain"), "Can't create symlink to domain\n");919 920 vmd_acpi_begin();921 922 pci_scan_child_bus(vmd->bus);923 vmd_domain_reset(vmd);924 925 /* When Intel VMD is enabled, the OS does not discover the Root Ports926 * owned by Intel VMD within the MMCFG space. pci_reset_bus() applies927 * a reset to the parent of the PCI device supplied as argument. This928 * is why we pass a child device, so the reset can be triggered at929 * the Intel bridge level and propagated to all the children in the930 * hierarchy.931 */932 list_for_each_entry(child, &vmd->bus->children, node) {933 if (!list_empty(&child->devices)) {934 dev = list_first_entry(&child->devices,935 struct pci_dev, bus_list);936 ret = pci_reset_bus(dev);937 if (ret)938 pci_warn(dev, "can't reset device: %d\n", ret);939 940 break;941 }942 }943 944 pci_assign_unassigned_bus_resources(vmd->bus);945 946 pci_walk_bus(vmd->bus, vmd_pm_enable_quirk, &features);947 948 /*949 * VMD root buses are virtual and don't return true on pci_is_pcie()950 * and will fail pcie_bus_configure_settings() early. It can instead be951 * run on each of the real root ports.952 */953 list_for_each_entry(child, &vmd->bus->children, node)954 pcie_bus_configure_settings(child);955 956 pci_bus_add_devices(vmd->bus);957 958 vmd_acpi_end();959 return 0;960}961 962static int vmd_probe(struct pci_dev *dev, const struct pci_device_id *id)963{964 unsigned long features = (unsigned long) id->driver_data;965 struct vmd_dev *vmd;966 int err;967 968 if (resource_size(&dev->resource[VMD_CFGBAR]) < (1 << 20))969 return -ENOMEM;970 971 vmd = devm_kzalloc(&dev->dev, sizeof(*vmd), GFP_KERNEL);972 if (!vmd)973 return -ENOMEM;974 975 vmd->dev = dev;976 vmd->instance = ida_alloc(&vmd_instance_ida, GFP_KERNEL);977 if (vmd->instance < 0)978 return vmd->instance;979 980 vmd->name = devm_kasprintf(&dev->dev, GFP_KERNEL, "vmd%d",981 vmd->instance);982 if (!vmd->name) {983 err = -ENOMEM;984 goto out_release_instance;985 }986 987 err = pcim_enable_device(dev);988 if (err < 0)989 goto out_release_instance;990 991 vmd->cfgbar = pcim_iomap(dev, VMD_CFGBAR, 0);992 if (!vmd->cfgbar) {993 err = -ENOMEM;994 goto out_release_instance;995 }996 997 pci_set_master(dev);998 if (dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(64)) &&999 dma_set_mask_and_coherent(&dev->dev, DMA_BIT_MASK(32))) {1000 err = -ENODEV;1001 goto out_release_instance;1002 }1003 1004 if (features & VMD_FEAT_OFFSET_FIRST_VECTOR)1005 vmd->first_vec = 1;1006 1007 spin_lock_init(&vmd->cfg_lock);1008 pci_set_drvdata(dev, vmd);1009 err = vmd_enable_domain(vmd, features);1010 if (err)1011 goto out_release_instance;1012 1013 dev_info(&vmd->dev->dev, "Bound to PCI domain %04x\n",1014 vmd->sysdata.domain);1015 return 0;1016 1017 out_release_instance:1018 ida_free(&vmd_instance_ida, vmd->instance);1019 return err;1020}1021 1022static void vmd_cleanup_srcu(struct vmd_dev *vmd)1023{1024 int i;1025 1026 for (i = 0; i < vmd->msix_count; i++)1027 cleanup_srcu_struct(&vmd->irqs[i].srcu);1028}1029 1030static void vmd_remove(struct pci_dev *dev)1031{1032 struct vmd_dev *vmd = pci_get_drvdata(dev);1033 1034 pci_stop_root_bus(vmd->bus);1035 sysfs_remove_link(&vmd->dev->dev.kobj, "domain");1036 pci_remove_root_bus(vmd->bus);1037 vmd_cleanup_srcu(vmd);1038 vmd_detach_resources(vmd);1039 vmd_remove_irq_domain(vmd);1040 ida_free(&vmd_instance_ida, vmd->instance);1041}1042 1043static void vmd_shutdown(struct pci_dev *dev)1044{1045 struct vmd_dev *vmd = pci_get_drvdata(dev);1046 1047 vmd_remove_irq_domain(vmd);1048}1049 1050#ifdef CONFIG_PM_SLEEP1051static int vmd_suspend(struct device *dev)1052{1053 struct pci_dev *pdev = to_pci_dev(dev);1054 struct vmd_dev *vmd = pci_get_drvdata(pdev);1055 int i;1056 1057 for (i = 0; i < vmd->msix_count; i++)1058 devm_free_irq(dev, vmd->irqs[i].virq, &vmd->irqs[i]);1059 1060 return 0;1061}1062 1063static int vmd_resume(struct device *dev)1064{1065 struct pci_dev *pdev = to_pci_dev(dev);1066 struct vmd_dev *vmd = pci_get_drvdata(pdev);1067 int err, i;1068 1069 vmd_set_msi_remapping(vmd, !!vmd->irq_domain);1070 1071 for (i = 0; i < vmd->msix_count; i++) {1072 err = devm_request_irq(dev, vmd->irqs[i].virq,1073 vmd_irq, IRQF_NO_THREAD,1074 vmd->name, &vmd->irqs[i]);1075 if (err)1076 return err;1077 }1078 1079 return 0;1080}1081#endif1082static SIMPLE_DEV_PM_OPS(vmd_dev_pm_ops, vmd_suspend, vmd_resume);1083 1084static const struct pci_device_id vmd_ids[] = {1085 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_VMD_201D),1086 .driver_data = VMD_FEAT_HAS_MEMBAR_SHADOW_VSCAP,},1087 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_VMD_28C0),1088 .driver_data = VMD_FEAT_HAS_MEMBAR_SHADOW |1089 VMD_FEAT_HAS_BUS_RESTRICTIONS |1090 VMD_FEAT_CAN_BYPASS_MSI_REMAP,},1091 {PCI_VDEVICE(INTEL, 0x467f),1092 .driver_data = VMD_FEATS_CLIENT,},1093 {PCI_VDEVICE(INTEL, 0x4c3d),1094 .driver_data = VMD_FEATS_CLIENT,},1095 {PCI_VDEVICE(INTEL, 0xa77f),1096 .driver_data = VMD_FEATS_CLIENT,},1097 {PCI_VDEVICE(INTEL, 0x7d0b),1098 .driver_data = VMD_FEATS_CLIENT,},1099 {PCI_VDEVICE(INTEL, 0xad0b),1100 .driver_data = VMD_FEATS_CLIENT,},1101 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_VMD_9A0B),1102 .driver_data = VMD_FEATS_CLIENT,},1103 {0,}1104};1105MODULE_DEVICE_TABLE(pci, vmd_ids);1106 1107static struct pci_driver vmd_drv = {1108 .name = "vmd",1109 .id_table = vmd_ids,1110 .probe = vmd_probe,1111 .remove = vmd_remove,1112 .shutdown = vmd_shutdown,1113 .driver = {1114 .pm = &vmd_dev_pm_ops,1115 },1116};1117module_pci_driver(vmd_drv);1118 1119MODULE_AUTHOR("Intel Corporation");1120MODULE_DESCRIPTION("Volume Management Device driver");1121MODULE_LICENSE("GPL v2");1122MODULE_VERSION("0.6");1123